This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
[PATCH] Add Requeue-PI support for x86 arch - V3
- From: Dinakar Guniguntala <dino at in dot ibm dot com>
- To: libc-alpha at sources dot redhat dot com
- Cc: Ulrich Drepper <drepper at redhat dot com>, Darren Hart <dvhltc at us dot ibm dot com>, Thomas Gleixner <tglx at linutronix dot de>
- Date: Wed, 2 Dec 2009 00:23:07 +0530
- Subject: [PATCH] Add Requeue-PI support for x86 arch - V3
- Reply-to: dino at in dot ibm dot com
Here is V3 of the requeue-PI for x86 arch. make check on nptl works fine now.
(nptl/tst-sem13.c fails but this happens on current git as well and is
unrelated to the changes below)
I have added the call to __pthread_mutex_cond_lock_adjust to ensure recursive
pi mutexes work fine. Also added the robust mutex changes which ensure that
the requeue-PI functions are called only for non robust PI mutexes.
The patch is against latest glibc git (2009-Dec-01)
-Dinakar
2009-12-01 Dinakar Guniguntala <dino@in.ibm.com>
* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.h: Define
FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: If mutex
is a non robust PI mutex, then use FUTEX_CMP_REQUEUE_PI.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: If mutex
is a non robust PI mutex, then use FUTEX_WAIT_REQUEUE_PI.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
diff -X ignore -Nurp glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
--- glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S 2009-11-30 09:08:07.000000000 -0800
+++ glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S 2009-11-30 09:34:53.000000000 -0800
@@ -91,12 +91,17 @@ __pthread_cond_broadcast:
8: cmpl $-1, %edi
je 9f
- /* XXX: The kernel so far doesn't support requeue to PI futex. */
- /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
- type of futex (private resp. shared). */
- testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi)
+ /* Do not use requeue for pshared condvars. */
+ testl $PS_BIT, MUTEX_KIND(%edi)
jne 9f
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edi), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ je 81f
+
/* Wake up all threads. */
#ifdef __ASSUME_PRIVATE_FUTEX
movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
@@ -138,6 +143,23 @@ __pthread_cond_broadcast:
cfi_restore_state
+81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $0x7fffffff, %esi
+ movl $1, %edx
+ /* Get the address of the futex involved. */
+# if MUTEX_FUTEX != 0
+ addl $MUTEX_FUTEX, %edi
+# endif
+ int $0x80
+
+ /* For any kind of error, which mainly is EAGAIN, we try again
+ with WAKE. The general test also covers running on old
+ kernels. */
+ cmpl $0xfffff001, %eax
+ jb 6b
+ jmp 9f
+
/* Initial locking failed. */
1:
#if cond_lock == 0
diff -X ignore -Nurp glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
--- glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S 2009-11-30 09:08:07.000000000 -0800
+++ glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S 2009-11-30 09:09:34.000000000 -0800
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <kernel-features.h>
+#include <pthread-pi-defines.h>
#include <pthread-errnos.h>
@@ -86,7 +87,17 @@ __pthread_cond_signal:
#endif
cmpl $-1, dep_mutex-cond_futex(%ebx)
sete %cl
- subl $1, %ecx
+ je 8f
+
+ movl dep_mutex-cond_futex(%ebx), %edx
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edx), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ je 9f
+
+8: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -124,8 +135,34 @@ __pthread_cond_signal:
cfi_restore_state
-7: /* %ecx should be either FUTEX_WAKE_OP or
- FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */
+9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $1, %edx
+ xorl %esi, %esi
+ movl dep_mutex-cond_futex(%ebx), %edi
+ movl (%ebx), %ebp
+ /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
+ sysenter.
+ ENTER_KERNEL */
+ int $0x80
+ popl %ebp
+ popl %esi
+
+ leal -cond_futex(%ebx), %edi
+
+ /* For any kind of error, we try again with WAKE.
+ The general test also covers running on old kernels. */
+ cmpl $-4095, %eax
+ jb 4f
+
+7:
+#ifdef __ASSUME_PRIVATE_FUTEX
+ andl $FUTEX_PRIVATE_FLAG, %ecx
+#else
+ andl %gs:PRIVATE_FUTEX, %ecx
+#endif
+ orl $FUTEX_WAKE, %ecx
+
xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx
movl $SYS_futex, %eax
/* %edx should be 1 already from $FUTEX_WAKE_OP syscall.
diff -X ignore -Nurp glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
--- glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S 2009-11-17 23:11:27.000000000 -0800
+++ glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S 2009-11-30 09:09:34.000000000 -0800
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -95,7 +96,7 @@ __pthread_cond_timedwait:
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 24
+#define FRAME_SIZE 32
subl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(FRAME_SIZE)
@@ -107,8 +108,10 @@ __pthread_cond_timedwait:
movl %edx, 16(%esp)
movl %eax, 20(%esp)
+ /* Reset the pi-requeued flag. */
+8: movl $0, 24(%esp)
/* Get the current time. */
-8: movl %ebx, %edx
+ movl %ebx, %edx
#ifdef __NR_clock_gettime
/* Get the clock number. */
movl cond_nwaiters(%ebx), %ebx
@@ -158,6 +161,7 @@ __pthread_cond_timedwait:
movl %edx, 8(%esp)
movl cond_futex(%ebx), %edi
+ movl %edi, 28(%esp)
/* Unlock. */
LOCK
@@ -172,13 +176,50 @@ __pthread_cond_timedwait:
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
- leal 4(%esp), %esi
#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 40f
+
+ movl dep_mutex(%ebx), %edi
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edi), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ jne 40f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ /* The following only works like this because we only support
+ two clocks, represented using a single bit. */
+ testl $1, cond_nwaiters(%ebx)
+ /* XXX Need to implement using sete instead of a jump. */
+ jne 42f
+ orl $FUTEX_CLOCK_REALTIME, %ecx
+
+ /* Requeue-PI uses absolute timeout */
+42: leal (%ebp), %esi
+ movl 28(%esp), %edx
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ movl %eax, %esi
+ /* Set the pi-requeued flag only if the kernel has returned 0. The
+ kernel does not hold the mutex on ETIMEDOUT or any other error. */
+ cmpl $0, %eax
+ sete 24(%esp)
+ je 41f
+
+ /* Normal and PI futexes dont mix. Use normal futex functions only
+ if the kernel does not support the PI futex functions. */
+ cmpl $-ENOSYS, %eax
+ jne 41f
+ xorl %ecx, %ecx
+
+40: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -187,7 +228,8 @@ __pthread_cond_timedwait:
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ leal 4(%esp), %esi
+ movl 28(%esp), %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
@@ -196,7 +238,7 @@ __pthread_cond_timedwait:
.Lsub_cond_futex:
movl %eax, %esi
- movl (%esp), %eax
+41: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
@@ -284,10 +326,16 @@ __pthread_cond_timedwait:
#endif
jne 10f
+11: xorl %eax, %eax
+ /* With requeue_pi, the mutex lock is held in the kernel. */
+ movl 24(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 26f
+
/* Remove cancellation handler. */
-11: movl 24+FRAME_SIZE(%esp), %eax
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+26: addl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(-FRAME_SIZE);
/* We return the result of the mutex_lock operation if it failed. */
@@ -317,6 +365,9 @@ __pthread_cond_timedwait:
cfi_restore_state
+27: call __pthread_mutex_cond_lock_adjust
+ jmp 26b
+
/* Initial locking failed. */
1:
#if cond_lock == 0
diff -X ignore -Nurp glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
--- glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S 2009-11-17 23:11:27.000000000 -0800
+++ glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S 2009-11-30 09:09:34.000000000 -0800
@@ -22,6 +22,8 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <tcb-offsets.h>
+#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -43,6 +45,9 @@ __pthread_cond_wait:
cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
#endif
+ pushl %ebp
+ cfi_adjust_cfa_offset(4)
+ cfi_rel_offset(%ebp, 0)
pushl %edi
cfi_adjust_cfa_offset(4)
cfi_rel_offset(%edi, 0)
@@ -55,7 +60,7 @@ __pthread_cond_wait:
cfi_remember_state
xorl %esi, %esi
- movl 16(%esp), %ebx
+ movl 20(%esp), %ebx
/* Get internal lock. */
movl $1, %edx
@@ -71,7 +76,7 @@ __pthread_cond_wait:
/* Store the reference to the mutex. If there is already a
different value in there this is a bad user bug. */
2: cmpl $-1, dep_mutex(%ebx)
- movl 20(%esp), %eax
+ movl 24(%esp), %eax
je 15f
movl %eax, dep_mutex(%ebx)
@@ -87,7 +92,7 @@ __pthread_cond_wait:
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 16
+#define FRAME_SIZE 20
subl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(FRAME_SIZE)
@@ -99,8 +104,10 @@ __pthread_cond_wait:
movl %edx, 8(%esp)
movl %eax, 12(%esp)
-8: movl cond_futex(%ebx), %edi
-
+ /* Reset the pi-requeued flag. */
+8: movl $0, 16(%esp)
+ movl cond_futex(%ebx), %ebp
+
/* Unlock. */
LOCK
#if cond_lock == 0
@@ -114,12 +121,39 @@ __pthread_cond_wait:
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
-#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
-#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 18f
+
+ movl dep_mutex(%ebx), %edi
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edi), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ jne 18f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl %ebp, %edx
+ xorl %esi, %esi
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ /* Set the pi-requeued flag only if the kernel has returned 0. The
+ kernel does not hold the mutex on error. */
+ cmpl $0, %eax
+ sete 16(%esp)
+ je 19f
+
+ /* Normal and PI futexes dont mix. Use normal futex functions only
+ if the kernel does not support the PI futex functions. */
+ cmpl $-ENOSYS, %eax
+ jne 19f
+ xorl %ecx, %ecx
+
+18: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -128,7 +162,7 @@ __pthread_cond_wait:
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ movl %ebp, %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
@@ -136,7 +170,7 @@ __pthread_cond_wait:
subl $cond_futex, %ebx
.Lsub_cond_futex:
- movl (%esp), %eax
+19: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
@@ -212,9 +246,15 @@ __pthread_cond_wait:
#endif
jne 10f
-11: movl 20+FRAME_SIZE(%esp), %eax
+ /* With requeue_pi, the mutex lock is held in the kernel. */
+11: xorl %eax, %eax
+ movl 16(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 20f
+
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+20: addl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(-FRAME_SIZE);
14: popl %ebx
@@ -226,12 +266,19 @@ __pthread_cond_wait:
popl %edi
cfi_adjust_cfa_offset(-4)
cfi_restore(%edi)
+ popl %ebp
+ cfi_adjust_cfa_offset(-4)
+ cfi_restore(%ebp)
/* We return the result of the mutex_lock operation. */
ret
cfi_restore_state
+21: call __pthread_mutex_cond_lock_adjust
+ xorl %eax, %eax
+ jmp 20b
+
/* Initial locking failed. */
1:
#if cond_lock == 0
@@ -484,7 +531,7 @@ __condvar_w_cleanup:
movl $0x7fffffff, %edx
ENTER_KERNEL
-5: movl 20+FRAME_SIZE(%esp), %eax
+5: movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
movl %esi, (%esp)
diff -X ignore -Nurp glibc/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
--- glibc/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h 2009-10-11 21:51:20.000000000 -0700
+++ glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h 2009-11-30 09:09:34.000000000 -0800
@@ -54,6 +54,8 @@
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
+#define FUTEX_WAIT_REQUEUE_PI 11
+#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256