This is the mail archive of the glibc-cvs@sourceware.org mailing list for the glibc project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

GNU C Library master sources branch dj/malloc updated. glibc-2.22-721-gd35e840


This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "GNU C Library master sources".

The branch, dj/malloc has been updated
       via  d35e84028cdade623838130dd4df3d0b22a13446 (commit)
       via  9c79af3a2ca600b99851cc25278d11b0b649f336 (commit)
      from  1322011d96a58f2d7cf9658eaa91f6645ff31b1a (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=d35e84028cdade623838130dd4df3d0b22a13446

commit d35e84028cdade623838130dd4df3d0b22a13446
Author: DJ Delorie <dj@delorie.com>
Date:   Fri Feb 19 17:10:24 2016 -0500

    Add trace2c script
    
    Usage: trace2c /tmp/mtrace.out > sample.c
    
    Converts a trace file to a compilable program that "somewhat"
    reflects the same malloc workload as the program which was
    traced.

diff --git a/malloc/trace2c b/malloc/trace2c
new file mode 100644
index 0000000..8d3eb74
--- /dev/null
+++ b/malloc/trace2c
@@ -0,0 +1,192 @@
+#!/usr/bin/perl
+# -*- perl -*-
+
+# Arrays starting with c_ are source code to be emitted later
+
+$last_idx = 0;
+sub ptr2idx {
+    my ($ptr) = @_;
+    if ($ptr2idx{$ptr}) {
+	return $ptr2idx{$ptr};
+    }
+    # we intentionally never return zero
+    $last_idx ++;
+    $ptr2idx{$ptr} = $last_idx;
+    push (@c_ptrvars, "volatile void *p$last_idx;");
+    return $last_idx;
+}
+
+$sync_counter = 0;
+
+# thread 2 waits for thread 1
+sub sync {
+    my ($thread1, $thread2) = @_;
+    if (! $sync_init{$thread1}) {
+	push (@c_sync, "volatile char sync_${thread1} = 0;");
+	$sync_init{$thread1} = 1;
+    }
+    $sync_counter ++;
+    push (@{$c_threads{$thread1}}, "  sync_${thread1} = $sync_counter;");
+    push (@{$c_threads{$thread1}}, "  __sync_synchronize ();");
+    push (@{$c_threads{$thread2}}, "  while (sync_${thread1} < $sync_counter)");
+    push (@{$c_threads{$thread2}}, "    __sync_synchronize ();");
+}
+
+sub acq_ptr {
+    my ($ptr) = @_;
+    if ($owner{$ptr} && $owner{$ptr} ne $thread) {
+	&sync ($owner{$ptr}, $thread);
+    }
+    $owner{$ptr} = $thread;
+}
+
+$master_thread = undef;
+
+$line = 0;
+while (<>) {
+    $line ++;
+    next if /^threadid/;
+    next if /out of/;
+
+    ($thread, $type, $path, $ptr1, $size, $ptr2) = split(' ');
+    $size = hex($size);
+    $idx1 = &ptr2idx($ptr1);
+    $idx2 = &ptr2idx($ptr2);
+
+    if (! $master_thread) {
+	$master_thread = $thread;
+    } elsif (! $threads{$thread}) {
+	# make new thread start at the "right" time
+	&sync ($master_thread, $thread);
+    }
+
+    $threads{$thread} = 1;
+
+    if ($type eq "malloc") {
+	# In case another thread needs to free this chunk first
+	&acq_ptr($ptr2);
+	push (@{$c_threads{$thread}}, sprintf ("  p%d = malloc (%d); // %d %s", $idx2, $size, $line, $ptr2));
+	push (@{$c_threads{$thread}}, sprintf ("  wmem (p%d, %d);", $idx2, $size));
+	$leak{$ptr2} = $size;
+	$owner{$ptr2} = $thread;
+	$valid{$ptr2} = 1;
+    }
+
+    if ($type eq "calloc") {
+	# In case another thread needs to free this chunk first
+	&acq_ptr($ptr2);
+	push (@{$c_threads{$thread}}, sprintf ("  p%d = calloc (%d,1); // %d %s", $idx2, $size, $line, $ptr2));
+	push (@{$c_threads{$thread}}, sprintf ("  wmem (p%d, %d);", $idx2, $size));
+	$leak{$ptr2} = $size;
+	$owner{$ptr2} = $thread;
+	$valid{$ptr2} = 1;
+    }
+
+    if ($type eq "free") {
+	if ($ptr1 =~ /^0+$/) {
+	    push (@{$c_threads{$thread}}, sprintf("  free (NULL);"));
+	} elsif ($valid{$ptr1}) {
+	    # if it was allocated in another thread
+	    &acq_ptr($ptr1);
+	    push (@{$c_threads{$thread}}, sprintf("  free ((void *)p%d); // %d %s", $idx1, $line, $ptr1));
+	    delete $leak{$ptr1};
+	    $valid{$ptr1} = 0;
+	} else {
+	    push (@{$c_threads{$thread}}, sprintf("  // free (p%s) (invalid ptr $ptr1 in thread $thread)", $idx1));
+	}
+    }
+
+    if ($type eq "realloc") {
+	if ($owner{$ptr1}) {
+	    &acq_ptr($ptr1);
+	    &acq_ptr($ptr2);
+	    push (@{$c_threads{$thread}}, sprintf("  p%d = realloc ((void *)p%d, %d);", $idx2, $idx1, $size));
+	    push (@{$c_threads{$thread}}, sprintf ("  wmem (p%d, %d);", $idx2, $size));
+	    # ptr1 might be the same as ptr2, so sequence matters
+	    delete $leak{$ptr1};
+	    $leak{$ptr2} = $size;
+	    $valid{$ptr1} = 0;
+	    $valid{$ptr2} = 1;
+	}
+    }
+}
+
+print '
+#include <stdio.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include <sys/time.h>
+
+static void
+wmem (volatile void *ptr, int count)
+{
+    char *p = (char *)ptr;
+    int i;
+    for (i=0; i<count; i+=8)
+	p[i] = 0;
+}
+';
+
+sub dump {
+    my ($x) = @_;
+    for ($i=0; $i<@{$x}; $i++) {
+	print ${$x}[$i], "\n";
+    }
+    print "\n";
+}
+&dump(\@c_ptrvars);
+&dump(\@c_sync);
+for $thread (sort keys %threads) {
+    print "pthread_t tid_$thread;\n";
+    print "void *thread_$thread(void *ignored) {\n";
+    &dump ($c_threads{$thread});
+    print "}\n\n";
+}
+
+for $p (sort keys %leak) {
+    #printf("// %16s %10d leaked\n", $p, $leak{$p});
+}
+
+print '
+
+static __inline__ int64_t rdtsc(void)
+{
+  unsigned a, d;
+  asm volatile("rdtsc" : "=a" (a), "=d" (d));
+  return ((unsigned long)a) | (((unsigned long)d) << 32);
+}
+
+int
+main()
+{
+    int64_t start;
+    int64_t end;
+    int64_t usec;
+    struct timeval tv_s, tv_e;
+
+    gettimeofday (&tv_s, NULL);
+    start = rdtsc();
+';
+
+for $thread (sort keys %threads) {
+  print "  pthread_create (&tid_$thread, NULL, thread_$thread, 0);\n";
+}
+for $thread (sort keys %threads) {
+  print "  pthread_join (tid_$thread, NULL);\n";
+}
+
+
+print '
+    end = rdtsc();
+    gettimeofday (&tv_e, NULL);
+    printf("%lld cycles\n", end - start);
+    if (tv_e.tv_usec < tv_s.tv_usec)
+	usec = (tv_e.tv_usec + 1000000 - tv_s.tv_usec) + (tv_e.tv_sec-1 - tv_s.tv_sec)*1000000;
+    else
+	usec = (tv_e.tv_usec - tv_s.tv_usec) + (tv_e.tv_sec - tv_s.tv_sec)*1000000;
+    printf("%lld usec\n", usec);
+    return 0;
+}
+';
+
+exit 0;

http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=9c79af3a2ca600b99851cc25278d11b0b649f336

commit 9c79af3a2ca600b99851cc25278d11b0b649f336
Author: DJ Delorie <dj@delorie.com>
Date:   Fri Feb 19 17:09:02 2016 -0500

    More trace hooks
    
    Add hooks to pvalloc and calloc
    
    Add path flag for when a call is handled via a hook function

diff --git a/malloc/malloc.c b/malloc/malloc.c
index 2fe4ada..69d62f6 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -1090,7 +1090,7 @@ volatile int __malloc_trace_buffer_head = 0;
 
 static __thread __malloc_trace_buffer_ptr trace_ptr;
 
-static inline void __attribute__((always_inline))
+static void
 __mtb_trace_entry (uint32_t type, int64_t size, void *ptr1)
 {
   int head1;
@@ -3031,7 +3031,10 @@ __libc_malloc (size_t bytes)
   void *(*hook) (size_t, const void *)
     = atomic_forced_read (__malloc_hook);
   if (__builtin_expect (hook != NULL, 0))
-    return (*hook)(bytes, RETURN_ADDRESS (0));
+    {
+      __MTB_TRACE_PATH (hook);
+      return (*hook)(bytes, RETURN_ADDRESS (0));
+    }
 
 #if USE_TCACHE
   if (bytes < MAX_TCACHE_SIZE)
@@ -3359,6 +3362,8 @@ __libc_valloc (size_t bytes)
 void *
 __libc_pvalloc (size_t bytes)
 {
+  void *rv;
+
   if (__malloc_initialized < 0)
     ptmalloc_init ();
 
@@ -3373,7 +3378,10 @@ __libc_pvalloc (size_t bytes)
       return 0;
     }
 
-  return _mid_memalign (pagesize, rounded_bytes, address);
+  __MTB_TRACE_ENTRY (PVALLOC, bytes, NULL);
+  rv = _mid_memalign (pagesize, rounded_bytes, address);
+  __MTB_TRACE_SET (ptr2, rv);
+  return rv;
 }
 
 void *
@@ -3389,6 +3397,7 @@ __libc_calloc (size_t n, size_t elem_size)
 
   /* size_t is unsigned so the behavior on overflow is defined.  */
   bytes = n * elem_size;
+  __MTB_TRACE_ENTRY (CALLOC, bytes, NULL);
 #define HALF_INTERNAL_SIZE_T \
   (((INTERNAL_SIZE_T) 1) << (8 * sizeof (INTERNAL_SIZE_T) / 2))
   if (__builtin_expect ((n | elem_size) >= HALF_INTERNAL_SIZE_T, 0))
@@ -3405,15 +3414,21 @@ __libc_calloc (size_t n, size_t elem_size)
   if (__builtin_expect (hook != NULL, 0))
     {
       sz = bytes;
+      __MTB_TRACE_PATH (hook);
       mem = (*hook)(sz, RETURN_ADDRESS (0));
       if (mem == 0)
+	{
+	  __MTB_TRACE_PATH (m_f_realloc);
         return 0;
+	}
 
+      __MTB_TRACE_SET (ptr2, mem);
       return memset (mem, 0, sz);
     }
 
   sz = bytes;
 
+  __MTB_TRACE_PATH (cpu_cache);
   arena_get (av, sz);
   if (av)
     {
@@ -3450,6 +3465,7 @@ __libc_calloc (size_t n, size_t elem_size)
 
   if (mem == 0 && av != NULL)
     {
+      __MTB_TRACE_PATH (cpu_cache2);
       LIBC_PROBE (memory_calloc_retry, 1, sz);
       av = arena_get_retry (av, sz);
       mem = _int_malloc (av, sz);
@@ -3463,6 +3479,7 @@ __libc_calloc (size_t n, size_t elem_size)
     return 0;
 
   p = mem2chunk (mem);
+  __MTB_TRACE_SET (ptr2, mem);
 
   /* Two optional cases in which clearing not necessary */
   if (chunk_is_mmapped (p))
diff --git a/malloc/mtrace-ctl.c b/malloc/mtrace-ctl.c
index 641a6a2..a253a56 100644
--- a/malloc/mtrace-ctl.c
+++ b/malloc/mtrace-ctl.c
@@ -56,6 +56,7 @@ const char * const typenames[] = {
   "realloc ",
   "memalign",
   "valloc  ",
+  "pvalloc  ",
 };
 
 void __attribute__((destructor))
@@ -80,7 +81,7 @@ djend()
 
   fprintf (outf, "%d out of %d events captured\n", head, size);
 
-  fprintf (outf, "threadid type     path    ptr1             size             ptr2\n");
+  fprintf (outf, "threadid type     path     ptr1             size             ptr2\n");
   for (i=0; i<size; i++)
     {
       __malloc_trace_buffer_ptr t = buf + (i+head) % size;
@@ -90,16 +91,17 @@ djend()
 	case __MTB_TYPE_UNUSED:
 	  break;
 	default:
-	  fprintf (outf, "%08x %s %d%d%d%d%d%d%d %016x %016x %016x\n",
+	  fprintf (outf, "%08x %s %c%c%c%c%c%c%c%c %016x %016x %016x\n",
 		   t->thread,
 		   typenames[t->type],
-		   t->path_thread_cache,
-		   t->path_cpu_cache,
-		   t->path_cpu_cache2,
-		   t->path_sbrk,
-		   t->path_mmap,
-		   t->path_munmap,
-		   t->path_m_f_realloc,
+		   t->path_thread_cache ? 'T' : '-',
+		   t->path_cpu_cache ? 'c' : '-',
+		   t->path_cpu_cache2 ? 'C' : '-',
+		   t->path_sbrk ? 's' : '-',
+		   t->path_mmap ? 'M' : '-',
+		   t->path_munmap ? 'U' : '-',
+		   t->path_m_f_realloc ? 'R' : '-',
+		   t->path_hook ? 'H' : '-',
 		   t->ptr1,
 		   t->size,
 		   t->ptr2);
diff --git a/malloc/mtrace.h b/malloc/mtrace.h
index e0fdd7b..a82d317 100644
--- a/malloc/mtrace.h
+++ b/malloc/mtrace.h
@@ -33,7 +33,8 @@ struct __malloc_trace_buffer_s {
   uint32_t path_mmap:1; /* mmap was called */
   uint32_t path_munmap:1; /* munmap was called */
   uint32_t path_m_f_realloc:1; /* realloc became malloc/free (i.e. next few records) */
-  uint32_t path:17; /* remaining bits */
+  uint32_t path_hook:1; /* A hook was used to complete the request */
+  uint32_t path:16; /* remaining bits */
 
   uint64_t ptr1;
   uint64_t ptr2;
@@ -85,3 +86,6 @@ void * __malloc_get_trace_buffer (int *bufcount, int *bufhead);
 
 /* ptr2 = valloc (size) */
 #define __MTB_TYPE_VALLOC	6
+
+/* ptr2 = pvalloc (size) */
+#define __MTB_TYPE_PVALLOC	7

-----------------------------------------------------------------------

Summary of changes:
 malloc/malloc.c     |   23 +++++-
 malloc/mtrace-ctl.c |   20 +++---
 malloc/mtrace.h     |    6 ++-
 malloc/trace2c      |  192 +++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 228 insertions(+), 13 deletions(-)
 create mode 100644 malloc/trace2c


hooks/post-receive
-- 
GNU C Library master sources


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]