separately compiling libc modules for rtld

[Roland McGrath <roland at redhat dot com>]

The following makefile changes implement a fully-automated means by which
the modules that heretofore have been linked into ld.so from libc_pic.a are
instead compiled separately for inclusion in ld.so, and so can be
conditionally compiled into different code in ld.so than what goes into
libc.  Having separately compiled modules is desireable for a few reasons:

The Hurd needs this to be able to get all the PLT entries it needs in ld.so
without sacrificing the PLT reduction in libc.so itself.  

When libc is built using TLS/__thread, we would like to have ld.so not use
any TLS accesses during startup, but libc_pic.a modules use TLS for errno.
This avoids the need for temporary thread pointer setup during bootstrap
and reduces the number of switch cases in the reloc handling for rtld
bootstrap.  More importantly, it lets prelinking ld.so work without
difficult new hackery for the TLS relocs.

It lets us use #if's freely in any libc code to minimize/optimize it
specially for the restricted uses in ld.so, rather than having to write
separate replacements.  e.g. the dl-xstat64.c kludge can go away because
the SHLIB_COMPAT macro can just be made not to fire for libc symbols in the
ld.so build.


These build changes per se should be a zero-change or close to it in the
compiled results.  (It provides a place where different build options could
be inserted, but nothing different is actually there yet.  I didn't compare
the compiled objects, but my builds with this work just the same as
before.)  But it is some untested new subtlety, so if 2.3 is very imminent
I don't know if this should wait for 2.3.1.  However, the Hurd is broken
now and without this can only be fixed with unacceptably ugly changes.  If
this goes in, I can do a very simple and clean fix for the Hurd's problems.
And it will open the gates for figuring out exactly how we want the TLS
build to be.


2002-09-30  Roland McGrath  <roland@redhat.com>

	* elf/rtld-Rules: New file.
	* elf/Makefile ($(objpfx)librtld.map, $(objpfx)librtld.mk,
	$(objpfx)rtld-libc.a): New targets.
	(generated): Add them.
	(reloc-link): Remove -o $@ from the variable.
	($(objpfx)dl-allobjs.os): Add -o $@ after $(reloc-link).
	(distribute): Add rtld-Rules.
	(CPPFLAGS-.os): Define this instead of CFLAGS-.os.
	* Makerules ($(+sysdir_pfx)sysd-rules): Emit rules for rtld-% targets.
	(common-mostlyclean, common-clean): Clean up rtld-* files.
	* sysdeps/unix/make-syscalls.sh: Add rtld-*.os target name to rules.

Index: Makerules
===================================================================
RCS file: /cvs/glibc/libc/Makerules,v
retrieving revision 1.366
diff -u -p -r1.366 Makerules
--- Makerules	23 Sep 2002 17:43:16 -0000	1.366
+++ Makerules	1 Oct 2002 05:50:55 -0000
@@ -235,16 +235,26 @@ $(+sysdir_pfx)sysd-rules: $(+sysdir_pfx)
 	     echo "\$$(objpfx)%$$o: $$dir/%.S \$$(before-compile); \
 		  \$$(compile-command.S)";				      \
 	     echo "\$$(objpfx)%$$o: $$dir/%.s \$$(before-compile); \
-		  \$$(compile-command.s)";				      \
+		  \$$(compile-command.s)";			              \
+	     echo "\$$(objpfx)rtld-%$$o: $$dir/%.S \$$(before-compile); \
+		  \$$(compile-command.S)";				      \
+	     echo "\$$(objpfx)rtld-%$$o: $$dir/%.s \$$(before-compile); \
+		  \$$(compile-command.s)";			              \
 	     $(close-check-inhibit-asm)	\
 	     echo "\$$(objpfx)%$$o: $$dir/%.c \$$(before-compile); \
 		  \$$(compile-command.c)";				      \
+	     echo "\$$(objpfx)rtld-%$$o: $$dir/%.c \$$(before-compile); \
+		  \$$(compile-command.c)";				      \
 	   done; \
 	   $(open-check-inhibit-asm) \
 	   echo "\$$(objpfx)%.d: $$dir/%.s \$$(common-objpfx)dummy.d; \
 		\$$(make-dummy-dep)";			       \
+	   echo "\$$(objpfx)rtld-%.d: $$dir/%.s \$$(common-objpfx)dummy.d; \
+		\$$(make-dummy-dep)";			       \
 	   echo "\$$(objpfx)%.d: $$dir/%.S \$$(before-compile); \
 		\$$(+make-deps)";					      \
+	   echo "\$$(objpfx)rtld-%.d: $$dir/%.S \$$(before-compile); \
+		\$$(+make-deps)";					      \
 	   $(close-check-inhibit-asm)	\
 	   echo "\$$(objpfx)%.d: $$dir/%.c \$$(before-compile); \
 		\$$(+make-deps)";					      \
@@ -1090,6 +1100,7 @@ common-mostlyclean:
 				     $(install-lib.so) \
 				     $(install-lib.so:%.so=%_pic.a))
 	-rm -f core
+	-rm -f $(objpfx)rtld-*.os
 	$(rmobjs)
 define rmobjs
 $(foreach o,$(object-suffixes-for-libc),
@@ -1099,6 +1110,7 @@ endef
 # Also remove the dependencies and generated source files.
 common-clean: common-mostlyclean
 	-rm -f $(addprefix $(objpfx),$(generated)) $(+depfiles)
+	-rm -f $(objpfx)rtld-*.d
 	-rm -fr $(addprefix $(objpfx),$(generated-dirs))
 	-rm -f $(addprefix $(common-objpfx),$(common-generated))
 	-rm -f $(objpfx)distinfo
Index: elf/Makefile
===================================================================
RCS file: /cvs/glibc/libc/elf/Makefile,v
retrieving revision 1.237
diff -u -p -r1.237 Makefile
--- elf/Makefile	23 Sep 2002 17:43:15 -0000	1.237
+++ elf/Makefile	1 Oct 2002 05:50:56 -0000
@@ -41,7 +41,8 @@ rtld-routines	:= rtld $(dl-routines) dl-
 		   dl-xstat64 dl-fxstat64
 all-rtld-routines = $(rtld-routines) $(sysdep-rtld-routines)
 
-distribute	:= $(rtld-routines:=.c) dynamic-link.h do-rel.h dl-machine.h \
+distribute	:= rtld-Rules \
+		   $(rtld-routines:=.c) dynamic-link.h do-rel.h dl-machine.h \
 		   dl-cache.h dl-hash.h soinit.c sofini.c ldd.bash.in \
 		   genrtldtbl.awk atomicity.h dl-procinfo.h ldsodefs.h \
 		   dl-librecon.h interp.c sln.c dl-dst.h hp-timing.h \
@@ -163,14 +164,49 @@ lib-noranlib: $(objpfx)$(rtld-installed-
 endif
 
 # Command to link into a larger single relocatable object.
-reloc-link = $(LINK.o) -nostdlib -nostartfiles -r -o $@
+reloc-link = $(LINK.o) -nostdlib -nostartfiles -r
 
 $(objpfx)dl-allobjs.os: $(all-rtld-routines:%=$(objpfx)%.os)
-	$(reloc-link) $^
+	$(reloc-link) -o $@ $^
 
 # Link together the dynamic linker into a single relocatable object.
-$(objpfx)librtld.os: $(objpfx)dl-allobjs.os $(common-objpfx)libc_pic.a
-	$(reloc-link) '-Wl,-(' $^ -lgcc '-Wl,-)'
+# First we do a link against libc_pic.a just to get a link map,
+# and discard the object produced by that link.  From the link map
+# we can glean all the libc modules that need to go into the dynamic
+# linker.  Then we do a recursive make that goes into all the subdirs
+# those modules come from and builds special rtld-foo.os versions that
+# are compiled with special flags, and puts these modules into rtld-libc.a
+# for us.  Then we do the real link using rtld-libc.a instead of libc_pic.a.
+
+$(objpfx)librtld.map: $(objpfx)dl-allobjs.os $(common-objpfx)libc_pic.a
+	$(reloc-link) -o $@.o '-Wl,-(' $^ -lgcc '-Wl,-)' -Wl,-Map,$@
+	rm -f $@.o
+
+$(objpfx)librtld.mk: $(objpfx)librtld.map Makefile
+	sed -n 's@^$(common-objpfx)\([^(]*\)(\(.*.os\))$$@\1 \2@p' $< | \
+	while read lib file; do \
+	  case $$lib in \
+	  libc_pic.a) \
+	    fgrep -l /$$file \
+		  $(common-objpfx)stamp.os $(common-objpfx)*/stamp.os | \
+	    sed 's@^$(common-objpfx)\([^/]*\)/stamp\.os$$@rtld-\1'" +=$$file@"\
+	    ;; \
+	  */*.a) \
+	    echo rtld-$${lib%%/*} += $$file ;; \
+	  *) echo "Wasn't expecting $$lib($$file)" >&2; exit 1 ;; \
+	  esac; \
+	done > $@T
+	echo rtld-subdirs = `sed 's/^rtld-\([^ ]*\).*$$/\1/' $@T \
+			     | sort -u` >> $@T
+	mv -f $@T $@
+
+$(objpfx)rtld-libc.a: $(objpfx)librtld.mk FORCE
+	$(MAKE) -f $< -f rtld-Rules
+
+generated += librtld.map librtld.mk rtld-libc.a
+
+$(objpfx)librtld.os: $(objpfx)dl-allobjs.os $(objpfx)rtld-libc.a
+	$(LINK.o) -nostdlib -nostartfiles -r -o $@ '-Wl,-(' $^ -lgcc '-Wl,-)'
 
 $(objpfx)ld.so: $(objpfx)librtld.os $(ld-map)
 	@rm -f $@.lds
@@ -206,6 +242,7 @@ $(objpfx)trusted-dirs.st: Makefile $(..)
 	$(make-target-directory)
 	echo "$(subst :, ,$(default-rpath) $(user-defined-trusted-dirs))"    \
 	| $(AWK) -f gen-trusted-dirs.awk > ${@:st=T};
+	echo '#define DL_DST_LIB "$(notdir $(slibdir))"' >> ${@:st=T}
 	$(move-if-change) ${@:st=T} ${@:st=h}
 	touch $@
 CPPFLAGS-dl-load.c = -I$(objpfx). -I$(csu-objpfx).
@@ -262,7 +299,7 @@ CFLAGS-ldconfig.c = $(SYSCONF-FLAGS) -D'
 CFLAGS-dl-cache.c = $(SYSCONF-FLAGS)
 CFLAGS-cache.c = $(SYSCONF-FLAGS)
 
-CFLAGS-.os += $(if $(filter $(@F),$(patsubst %,%.os,$(all-rtld-routines))),-DNOT_IN_libc -DIS_IN_rtld)
+CPPFLAGS-.os += $(if $(filter $(@F),$(patsubst %,%.os,$(all-rtld-routines))),-DNOT_IN_libc -DIS_IN_rtld)
 
 test-modules = $(addprefix $(objpfx),$(addsuffix .so,$(strip $(modules-names))))
 generated += $(addsuffix .so,$(strip $(modules-names)))
Index: elf/rtld-Rules
===================================================================
RCS file: elf/rtld-Rules
diff -N elf/rtld-Rules
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ elf/rtld-Rules	1 Oct 2002 05:50:56 -0000
@@ -0,0 +1,100 @@
+# Subroutine makefile for compiling libc modules linked into dynamic linker.
+
+# Copyright (C) 2002 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, write to the Free
+# Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# 02111-1307 USA.
+
+# This makefile is never used by itself, but only from the rtld-libc.a
+# rule in Makefile, which does make -f librtld.mk -f rtld-Rules.
+# librtld.mk is the generated file containing variable definitions for
+# `rtld-subdirs', a subset of the top-level $(subdirs) list; and for each
+# SUBDIR in $(rtld-subdirs), `rtld-SUBDIR' listing `module.os' file names.
+
+.PHONY: rtld-all
+rtld-all:
+
+# When run from the elf/Makefile to build rtld-libc.a, $(subdir) is elf.
+ifeq ($(subdir),elf)
+
+ifndef rtld-subdirs
+error This is makefile is a subroutine of elf/Makefile not to be used directly
+endif
+
+include ../Makeconfig
+
+rtld-all: $(objpfx)rtld-libc.a
+
+$(objpfx)rtld-libc.a: $(foreach dir,$(rtld-subdirs),\
+				$(addprefix $(common-objpfx)$(dir)/rtld-,\
+					    $(rtld-$(dir))))
+	@-rm -f $@T
+	$(AR) cq $@T $^
+	$(RANLIB) $@T
+	mv -f $@T $@
+
+# For each subdirectory, define a pattern rule that makes all of that
+# subdirectory's modules at once with one recursive make command.
+object-suffixes-left := $(rtld-subdirs)
+define o-iterator-doit
+$(foreach obj,$(rtld-$o),$(common-objpfx)%/rtld-$(obj)): FORCE ; \
+	+$$(rtld-subdir-make)
+endef
+include $(patsubst %,../o-iterator.mk,$(object-suffixes-left))
+
+# This is how we descend into each subdirectory.  See below.
+define rtld-subdir-make
+$(MAKE) -C ../$* objdir=$(objdir) -f Makefile -f ../elf/rtld-Rules rtld-all \
+	rtld-modules='$(addprefix rtld-,$(rtld-$*))'
+endef
+
+FORCE:
+
+else
+
+# In this case we are being run by $(rtld-subdir-make), above.
+# Some other subdir's Makefile has provided all its normal rules,
+# and we just provide some additional definitions.
+
+# These are the basic compilation rules corresponding to the Makerules ones.
+# The sysd-rules generated makefile already defines pattern rules for rtld-%
+# targets built from sysdeps source files.
+$(objpfx)rtld-%.os: %.S $(before-compile); $(compile-command.S)
+$(objpfx)rtld-%.d: %.S $(before-compile); $(+make-deps)
+$(objpfx)rtld-%.os: %.s $(before-compile); $(compile-command.s)
+$(objpfx)rtld-%.d: %.s $(before-compile); $(+make-deps)
+$(objpfx)rtld-%.os: %.c $(before-compile); $(compile-command.c)
+$(objpfx)rtld-%.d: %.c $(before-compile); $(+make-deps)
+
+# The command line setting of rtld-modules (see above) tells us
+# what we need to build, and that tells us what dependency files we need.
+rtld-all: $(addprefix $(objpfx),$(rtld-modules))
+
+-include $(rtld-modules:%.os=$(objpfx)%.d)
+
+# Just in case we wind up e.g. regenerating dependencies for non-rtld objects,
+# we don't unconditionally modify the flags.  For rtld-% targets, use the
+# special flags set below.
+CPPFLAGS += $(patsubst rtld-%,$(CPPFLAGS-rtld),$(@F))
+CFLAGS   += $(patsubst rtld-%,$(CFLAGS-rtld),$(@F))
+
+
+# This here is the whole point of all the shenanigans.
+CPPFLAGS-rtld := -DNOT_IN_libc -DIS_IN_rtld
+CFLAGS-rtld := # blah
+
+
+endif
Index: sysdeps/unix/make-syscalls.sh
===================================================================
RCS file: /cvs/glibc/libc/sysdeps/unix/make-syscalls.sh,v
retrieving revision 1.26
diff -u -p -r1.26 make-syscalls.sh
--- sysdeps/unix/make-syscalls.sh	5 Aug 2002 06:57:52 -0000	1.26
+++ sysdeps/unix/make-syscalls.sh	1 Oct 2002 05:50:56 -0000
@@ -128,7 +128,8 @@ shared-only-routines += $file
     ;;
   *)
     echo "\
-\$(foreach o,\$(object-suffixes),\$(objpfx)$file\$o): \\"
+\$(foreach o,\$(object-suffixes),\$(objpfx)$file\$o) \
+\$(objpfx)rtld-$file.os: \\"
     ;;
   esac