[RFC] [PATCH 2/2] nptl: Enable pthread rwlock to use the TP futex

[Waiman Long <longman at redhat dot com>]

This patch adds a new lock kind (PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
to the pthread_rwlockattr_setkind_np() to enable the pthread rwlock
to use the TP futex, if available, for supporting rwlock.

Signed-off-by: Waiman Long <longman@redhat.com>
---
 ChangeLog                                    |  26 +++
 nptl/pthread_rwlock_rdlock.c                 |   5 +-
 nptl/pthread_rwlock_timedrdlock.c            |   5 +-
 nptl/pthread_rwlock_timedwrlock.c            |   5 +-
 nptl/pthread_rwlock_tp.c                     | 235 +++++++++++++++++++++++++++
 nptl/pthread_rwlock_tryrdlock.c              |   5 +
 nptl/pthread_rwlock_trywrlock.c              |   5 +
 nptl/pthread_rwlock_unlock.c                 |  14 ++
 nptl/pthread_rwlock_wrlock.c                 |   5 +-
 nptl/pthread_rwlockattr_setkind_np.c         |  21 ++-
 sysdeps/nptl/pthread.h                       |   1 +
 sysdeps/unix/sysv/linux/lowlevellock-futex.h |   7 +
 12 files changed, 329 insertions(+), 5 deletions(-)
 create mode 100644 nptl/pthread_rwlock_tp.c

diff --git a/ChangeLog b/ChangeLog
index ea655fc..49255db 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,29 @@
+2017-10-24  Waiman Long  <longman@redhat.com>
+
+	* nptl/pthreadP.h: Add pthread mutex macros to support TP futexes.
+	* nptl/pthread_mutex_init.c: Add test for the presence of TP futexes.
+	* nptl/pthread_mutex_lock.c: Add TP futexes support.
+	* nptl/pthread_mutex_timedlock.c: Add TP futexes support.
+	* nptl/pthread_mutex_trylock.c: Add TP futexes support.
+	* nptl/pthread_mutex_unlock.c: Add TP futexes support.
+	* nptl/pthread_mutexattr_setprotocol.c: Make it accept a new protocol
+	PTHREAD_THROUGHPUT_NP.
+	* nptl/pthread_rwlock_rdlock.c: Add TP futexes support.
+	* nptl/pthread_rwlock_timedrdlock.c: Add TP futexes support.
+	* nptl/pthread_rwlock_timedwrlock.c: Add TP futexes support.
+	* nptl/pthread_rwlock_tp.c: New file to make TP futex syscalls.
+	* nptl/pthread_rwlock_tryrdlock.c: Add TP futexes support.
+	* nptl/pthread_rwlock_trywrlock.c: Add TP futexes support.
+	* nptl/pthread_rwlock_unlock.c: Add TP futexes support.
+	* nptl/pthread_rwlock_wrlock.c: Add TP futexes support.
+	* nptl/pthread_rwlockattr_setkind_np.c: Make it accept a new kind
+	PTHREAD_RWLOCK_USE_TP_FUTEX_NP.
+	* sysdeps/nptl/pthread.h: Add PTHREAD_THROUGHPUT_NP &
+	PTHREAD_RWLOCK_USE_TP_FUTEX_NP.
+	* sysdeps/unix/sysv/linux/hppa/pthread.h: Add PTHREAD_THROUGHPUT_NP.
+	* sysdeps/unix/sysv/linux/lowlevellock-futex.h: Add new TP futexes
+	related macros.
+
 2017-10-19  Valery Reznic <valery_reznic@yahoo.com>
 	    H.J. Lu  <hongjiu.lu@intel.com>
 
diff --git a/nptl/pthread_rwlock_rdlock.c b/nptl/pthread_rwlock_rdlock.c
index e07581b..fe408b1 100644
--- a/nptl/pthread_rwlock_rdlock.c
+++ b/nptl/pthread_rwlock_rdlock.c
@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
 
 /* See pthread_rwlock_common.c.  */
 int
@@ -24,7 +25,9 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
 {
   LIBC_PROBE (rdlock_entry, 1, rwlock);
 
-  int result = __pthread_rwlock_rdlock_full (rwlock, NULL);
+  int result = (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+	     ? __pthread_rwlock_rdlock_tp   (rwlock, NULL)
+	     : __pthread_rwlock_rdlock_full (rwlock, NULL);
   LIBC_PROBE (rdlock_acquire_read, 1, rwlock);
   return result;
 }
diff --git a/nptl/pthread_rwlock_timedrdlock.c b/nptl/pthread_rwlock_timedrdlock.c
index 9f084f8..d818184 100644
--- a/nptl/pthread_rwlock_timedrdlock.c
+++ b/nptl/pthread_rwlock_timedrdlock.c
@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
 
 /* See pthread_rwlock_common.c.  */
 int
@@ -33,5 +34,7 @@ pthread_rwlock_timedrdlock (pthread_rwlock_t *rwlock,
       || abstime->tv_nsec < 0))
     return EINVAL;
 
-  return __pthread_rwlock_rdlock_full (rwlock, abstime);
+  return (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+	 ? __pthread_rwlock_rdlock_tp   (rwlock, abstime)
+	 : __pthread_rwlock_rdlock_full (rwlock, abstime);
 }
diff --git a/nptl/pthread_rwlock_timedwrlock.c b/nptl/pthread_rwlock_timedwrlock.c
index 5626505..e1dc3a4 100644
--- a/nptl/pthread_rwlock_timedwrlock.c
+++ b/nptl/pthread_rwlock_timedwrlock.c
@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
 
 /* See pthread_rwlock_common.c.  */
 int
@@ -33,5 +34,7 @@ pthread_rwlock_timedwrlock (pthread_rwlock_t *rwlock,
       || abstime->tv_nsec < 0))
     return EINVAL;
 
-  return __pthread_rwlock_wrlock_full (rwlock, abstime);
+  return (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+	 ? __pthread_rwlock_wrlock_tp   (rwlock, abstime)
+	 : __pthread_rwlock_wrlock_full (rwlock, abstime);
 }
diff --git a/nptl/pthread_rwlock_tp.c b/nptl/pthread_rwlock_tp.c
new file mode 100644
index 0000000..1ecde7b
--- /dev/null
+++ b/nptl/pthread_rwlock_tp.c
@@ -0,0 +1,235 @@
+/* POSIX reader--writer lock: TP futex specific code
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <errno.h>
+#include <sysdep.h>
+#include <pthread.h>
+#include <pthreadP.h>
+#include <sys/time.h>
+#include <stap-probe.h>
+#include <atomic.h>
+#include <futex-internal.h>
+
+/*
+ * The __writers_futex is used as the target of the TP futex syscalls.
+ */
+
+#ifdef __NR_futex
+
+#define FUTEX_FLAGS_MASK	(~FUTEX_TID_MASK)
+#define lll_futex_tp_lock(futexp, val, timeout, private)		\
+	lll_futex_syscall(4, futexp,					\
+			  __lll_private_flag(FUTEX_LOCK, private),	\
+			  val, timeout)
+#define lll_futex_tp_unlock(futexp, private)				\
+	lll_futex_syscall(4, futexp,					\
+			  __lll_private_flag(FUTEX_UNLOCK, private),	\
+			  0, NULL)
+#define lll_futex_tp_lock_shared(futexp, timeout, private)		\
+	lll_futex_syscall(4, futexp,					\
+			  __lll_private_flag(FUTEX_LOCK_SHARED, private),\
+			  0, timeout)
+#define lll_futex_tp_unlock_shared(futexp, private)			\
+	lll_futex_syscall(4, futexp,					\
+			  __lll_private_flag(FUTEX_UNLOCK_SHARED, private),\
+			  0, NULL)
+
+static __always_inline int
+__pthread_rwlock_private (pthread_rwlock_t *rwlock)
+{
+  return rwlock->__data.__shared != 0 ? FUTEX_SHARED : FUTEX_PRIVATE;
+}
+
+/* Return 1 if the lock acquired, 0 otherwise */
+static __always_inline int
+__pthread_rwlock_tryrdlock_tp(pthread_rwlock_t *rwlock)
+{
+  unsigned int *futex = &rwlock->__data.__writers_futex;
+  unsigned int oldval = atomic_compare_and_exchange_val_acq
+			(futex, FUTEX_SHARED_FLAG + 1, 0);
+  if (oldval == 0)
+    return 1;
+
+  while (1)
+    {
+      unsigned int new, old = oldval;
+
+      /*
+       * Try to increment the reader count only if
+       * 1) the FUTEX_SHARED_FLAG bit is set; and
+       * 2) none of the flags bits or FUTEX_SHARED_UNLOCK is set.
+       */
+      if (!old)
+	new = FUTEX_SHARED_FLAG + 1;
+      else if ((old & FUTEX_SHARED_FLAG) &&
+	      !(old & (FUTEX_FLAGS_MASK|FUTEX_SHARED_UNLOCK)))
+	new = old + 1;
+      else
+	return 0;
+
+      oldval = atomic_compare_and_exchange_val_acq(futex, new, old);
+      if (old == oldval)
+	return 1;
+    }
+    /* Unreachable */
+}
+
+/* Return 1 if the lock acquired, 0 otherwise */
+static __always_inline int
+__pthread_rwlock_trywrlock_tp(pthread_rwlock_t *rwlock)
+{
+  pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
+  unsigned int *futex = &rwlock->__data.__writers_futex;
+
+  return atomic_compare_and_exchange_val_acq(futex, id, 0) == 0;
+}
+
+static __always_inline int
+__pthread_rwlock_rdlock_tp(pthread_rwlock_t *rwlock,
+			   const struct timespec *abstime)
+{
+  int private = __pthread_rwlock_private(rwlock);
+  unsigned int *futex = &rwlock->__data.__writers_futex;
+
+  if (__pthread_rwlock_tryrdlock_tp(rwlock))
+    return 0;
+
+  return lll_futex_tp_lock_shared(futex, abstime, private);
+}
+
+static __always_inline int
+__pthread_rwlock_wrlock_tp(pthread_rwlock_t *rwlock,
+			   const struct timespec *abstime)
+{
+  pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
+  unsigned int *futex = &rwlock->__data.__writers_futex;
+  int private = __pthread_rwlock_private(rwlock);
+  int uslockcnt = 4;	/* Do 4 userspace locks before doing kernel lock */
+
+  if (__pthread_rwlock_trywrlock_tp(rwlock))
+    return 0;
+
+  while (1)
+    {
+      int err = lll_futex_tp_lock(futex, uslockcnt, abstime, private);
+
+      /* It is possible, though extremely unlikely, that a lock
+	 handoff has happened and an error is returned. So we
+	 need to check for that here.  */
+      if (!err || ((*futex & FUTEX_TID_MASK) == id))
+	return 0;
+
+      if (err == ETIMEDOUT)
+	return err;
+
+      if ((err == EAGAIN) && __pthread_rwlock_trywrlock_tp(rwlock))
+	return 0;
+
+      uslockcnt--;
+    }
+    /* Unreachable */
+}
+
+static __always_inline void
+__pthread_rwlock_rdunlock_tp(pthread_rwlock_t *rwlock)
+{
+  int val = atomic_fetch_add_release(&rwlock->__data.__writers_futex, -1) - 1;
+  unsigned int *futex = &rwlock->__data.__writers_futex;
+  int private = __pthread_rwlock_private(rwlock);
+
+  while (1)
+    {
+      int oldval;
+
+      /* Return if not the last reader, not in shared locking
+	 mode or the unlock bit has been set.  */
+      if ((val & (FUTEX_SCNT_MASK|FUTEX_SHARED_UNLOCK)) ||
+	 !(val & FUTEX_SHARED_FLAG))
+	return;
+
+      if (val & FUTEX_FLAGS_MASK)
+	{
+	  /* Only one task that can set the FUTEX_SHARED_UNLOCK
+	     bit will do the unlock to wake up the waiters. */
+	  oldval = atomic_compare_and_exchange_val_rel
+			(futex, val|FUTEX_SHARED_UNLOCK, val);
+	  if (oldval == val)
+	    break;
+	}
+      else
+	{
+	  /* Try to clear the futex when there is no waiter. */
+	  oldval = atomic_compare_and_exchange_val_rel(futex, 0, val);
+	  if (oldval == val)
+	    return;
+	}
+	val = oldval;
+    }
+
+  lll_futex_tp_unlock_shared(futex, private);
+}
+
+static __always_inline void
+__pthread_rwlock_wrunlock_tp(pthread_rwlock_t *rwlock)
+{
+  pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
+  unsigned int *futex = &rwlock->__data.__writers_futex;
+  int private = __pthread_rwlock_private(rwlock);
+
+  if (atomic_compare_and_exchange_val_rel(futex, 0, id) == id)
+    return;
+
+  lll_futex_tp_unlock(futex, private);
+}
+
+#else /* __NR_futex */
+
+static __always_inline int
+__pthread_rwlock_tryrdlock_tp(pthread_rwlock_t *rwlock)
+{
+}
+
+static __always_inline int
+__pthread_rwlock_trywrlock_tp(pthread_rwlock_t *rwlock)
+{
+}
+
+
+static __always_inline void
+__pthread_rwlock_rdlock_tp(pthread_rwlock_t *rwlock,
+				const struct timespec *abstime)
+{
+}
+
+static __always_inline void
+__pthread_rwlock_wrlock_tp(pthread_rwlock_t *rwlock,
+				const struct timespec *abstime)
+{
+}
+
+static __always_inline void
+__pthread_rwlock_rdunlock_tp(pthread_rwlock_t *rwlock)
+{
+}
+
+static __always_inline void
+__pthread_rwlock_wrunlock_tp(pthread_rwlock_t *rwlock)
+{
+}
+
+#endif /* __NR_futex */
diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c
index 6c3014c..11f3a16 100644
--- a/nptl/pthread_rwlock_tryrdlock.c
+++ b/nptl/pthread_rwlock_tryrdlock.c
@@ -21,6 +21,7 @@
 #include <atomic.h>
 #include <stdbool.h>
 #include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
 
 
 /* See pthread_rwlock_common.c for an overview.  */
@@ -39,6 +40,10 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
      below.  */
   unsigned int r = atomic_load_relaxed (&rwlock->__data.__readers);
   unsigned int rnew;
+
+  if (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+    return __pthread_rwlock_tryrdlock_tp(rwlock) ? 0 : EBUSY;
+
   do
     {
       if ((r & PTHREAD_RWLOCK_WRPHASE) == 0)
diff --git a/nptl/pthread_rwlock_trywrlock.c b/nptl/pthread_rwlock_trywrlock.c
index 0d9ccaf..1ca4c3a 100644
--- a/nptl/pthread_rwlock_trywrlock.c
+++ b/nptl/pthread_rwlock_trywrlock.c
@@ -19,6 +19,7 @@
 #include <errno.h>
 #include "pthreadP.h"
 #include <atomic.h>
+#include "pthread_rwlock_tp.c"
 
 /* See pthread_rwlock_common.c for an overview.  */
 int
@@ -37,6 +38,10 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
   unsigned int r = atomic_load_relaxed (&rwlock->__data.__readers);
   bool prefer_writer =
       (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP);
+
+  if (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+      return  __pthread_rwlock_trywrlock_tp(rwlock) ? 0 : EBUSY;
+
   while (((r & PTHREAD_RWLOCK_WRLOCKED) == 0)
       && (((r >> PTHREAD_RWLOCK_READER_SHIFT) == 0)
 	  || (prefer_writer && ((r & PTHREAD_RWLOCK_WRPHASE) != 0))))
diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c
index ef46e88..153c2e2 100644
--- a/nptl/pthread_rwlock_unlock.c
+++ b/nptl/pthread_rwlock_unlock.c
@@ -24,6 +24,7 @@
 #include <stap-probe.h>
 
 #include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
 
 /* See pthread_rwlock_common.c for an overview.  */
 int
@@ -31,6 +32,19 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
 {
   LIBC_PROBE (rwlock_unlock, 1, rwlock);
 
+  if (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+    {
+      unsigned int *futex = &rwlock->__data.__writers_futex;
+
+      /* For TP futex, the FUTEX_SHARED bit is used to distinguish between a
+         writer and reader-owned lock. */
+      if (atomic_load_relaxed (futex) & FUTEX_SHARED_FLAG)
+	__pthread_rwlock_rdunlock_tp (rwlock);
+      else
+	__pthread_rwlock_wrunlock_tp (rwlock);
+      return 0;
+    }
+
   /* We distinguish between having acquired a read vs. a write lock by looking
      at the writer TID.  If it's equal to our TID, we must be the writer
      because nobody else can have stored this value.  Also, if we are a
diff --git a/nptl/pthread_rwlock_wrlock.c b/nptl/pthread_rwlock_wrlock.c
index 335fcd1..acaee21 100644
--- a/nptl/pthread_rwlock_wrlock.c
+++ b/nptl/pthread_rwlock_wrlock.c
@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
 
 /* See pthread_rwlock_common.c.  */
 int
@@ -24,7 +25,9 @@ __pthread_rwlock_wrlock (pthread_rwlock_t *rwlock)
 {
   LIBC_PROBE (wrlock_entry, 1, rwlock);
 
-  int result = __pthread_rwlock_wrlock_full (rwlock, NULL);
+  int result = (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+	     ? __pthread_rwlock_wrlock_tp   (rwlock, NULL)
+	     : __pthread_rwlock_wrlock_full (rwlock, NULL);
   LIBC_PROBE (wrlock_acquire_write, 1, rwlock);
   return result;
 }
diff --git a/nptl/pthread_rwlockattr_setkind_np.c b/nptl/pthread_rwlockattr_setkind_np.c
index b3cdc7f..2d35f8b 100644
--- a/nptl/pthread_rwlockattr_setkind_np.c
+++ b/nptl/pthread_rwlockattr_setkind_np.c
@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include <errno.h>
+#include <assert.h>
 #include "pthreadP.h"
 
 
@@ -25,7 +26,25 @@ pthread_rwlockattr_setkind_np (pthread_rwlockattr_t *attr, int pref)
 {
   struct pthread_rwlockattr *iattr;
 
-  if (pref != PTHREAD_RWLOCK_PREFER_READER_NP
+  if (pref == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+    {
+#ifdef __NR_futex
+      static int tp_futex_supported;
+      if (tp_futex_supported == 0)
+	{
+	  int lock = 0;
+	  INTERNAL_SYSCALL_DECL (err);
+	  int ret = INTERNAL_SYSCALL (futex, err, 4, &lock, FUTEX_UNLOCK_SHARED, 0, 0);
+	  assert (INTERNAL_SYSCALL_ERROR_P (ret, err));
+	  tp_futex_supported = INTERNAL_SYSCALL_ERRNO (ret, err) == ENOSYS ? -1 : 1;
+	}
+      if (tp_futex_supported < 0)
+	return ENOTSUP;
+#else
+      return ENOTSUP;
+#endif
+    }
+  else if (pref != PTHREAD_RWLOCK_PREFER_READER_NP
       && pref != PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP
       && __builtin_expect  (pref != PTHREAD_RWLOCK_PREFER_WRITER_NP, 0))
     return EINVAL;
diff --git a/sysdeps/nptl/pthread.h b/sysdeps/nptl/pthread.h
index b2fe9e6..a1e88c9 100644
--- a/sysdeps/nptl/pthread.h
+++ b/sysdeps/nptl/pthread.h
@@ -120,6 +120,7 @@ enum
   PTHREAD_RWLOCK_PREFER_READER_NP,
   PTHREAD_RWLOCK_PREFER_WRITER_NP,
   PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP,
+  PTHREAD_RWLOCK_USE_TP_FUTEX_NP,
   PTHREAD_RWLOCK_DEFAULT_NP = PTHREAD_RWLOCK_PREFER_READER_NP
 };
 
diff --git a/sysdeps/unix/sysv/linux/lowlevellock-futex.h b/sysdeps/unix/sysv/linux/lowlevellock-futex.h
index 89b1674..b093d9b 100644
--- a/sysdeps/unix/sysv/linux/lowlevellock-futex.h
+++ b/sysdeps/unix/sysv/linux/lowlevellock-futex.h
@@ -40,11 +40,18 @@
 #define FUTEX_CMP_REQUEUE_PI    12
 #define FUTEX_LOCK		13
 #define FUTEX_UNLOCK		14
+#define FUTEX_LOCK_SHARED	15
+#define FUTEX_UNLOCK_SHARED	16
 #define FUTEX_PRIVATE_FLAG	128
 #define FUTEX_CLOCK_REALTIME	256
 
 #define FUTEX_BITSET_MATCH_ANY	0xffffffff
 
+/* Bits used by TP futex */
+#define FUTEX_SHARED_FLAG	0x20000000
+#define FUTEX_SHARED_UNLOCK	0x10000000
+#define FUTEX_SCNT_MASK 	0x00ffffff
+
 /* Values for 'private' parameter of locking macros.  Yes, the
    definition seems to be backwards.  But it is not.  The bit will be
    reversed before passing to the system call.  */
-- 
1.8.3.1