[PATCH] malloc: fix local_next handling

[Joern Engel <joern at purestorage dot com>]

arena_get2() still used the old ->next pointer to find the next arena -
ignoring numa locality.  Initial per-node arenas were not on the global
list.

Also, change the numa_arena[node] pointers whenever we use them.
Otherwise those arenas become hot spots.  If we move them around, the
load gets spread roughly evenly.

JIRA: PURE-27597
---
 tpc/malloc2.13/arena.h | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/tpc/malloc2.13/arena.h b/tpc/malloc2.13/arena.h
index 0ee6286bf286..599774ea1300 100644
--- a/tpc/malloc2.13/arena.h
+++ b/tpc/malloc2.13/arena.h
@@ -688,11 +688,18 @@ static struct malloc_state *_int_new_arena(size_t size, int numa_node)
 	(void)mutex_lock(&a->mutex);
 
 
-	/* Add the new arena to the global list.  */
+	/* Add the new arena to the global lists.  */
+	a->numa_node = numa_node;
+
 	a->next = main_arena.next;
 	atomic_write_barrier();
 	main_arena.next = a;
-	a->numa_node = numa_node;
+
+	if (numa_arena[numa_node]) {
+		a->local_next = numa_arena[numa_node]->local_next;
+		atomic_write_barrier();
+		numa_arena[numa_node]->local_next = a;
+	}
 
 	THREAD_STAT(++(a->stat_lock_loop));
 
@@ -703,7 +710,7 @@ static struct malloc_state *arena_get2(struct malloc_state *a_tsd, size_t size)
 {
 	struct malloc_state *a;
 
-	a = a_tsd->next;
+	a = a_tsd->local_next;
 	if (!a)
 		abort();
 
@@ -718,7 +725,7 @@ static struct malloc_state *arena_get2(struct malloc_state *a_tsd, size_t size)
 			tsd_setspecific(arena_key, (Void_t *) a);
 			return a;
 		}
-		a = a->next;
+		a = a->local_next;
 	} while (a != a_tsd);
 
 	/* If not even the list_lock can be obtained, try again.  This can
@@ -768,8 +775,10 @@ static struct malloc_state *arena_get(size_t size)
 	int node = getnode();
 
 	tsd_getspecific(arena_key, arena);
-	if (!arena || arena->numa_node != node)
+	if (!arena || arena->numa_node != node) {
 		arena = numa_arena[node];
+		numa_arena[node] = arena->local_next;
+	}
 	if (arena && !mutex_trylock(&arena->mutex)) {
 		THREAD_STAT(++(arena->stat_lock_direct));
 	} else
-- 
2.7.0.rc3