Refactor the build system for easily adding crates

Before this patch, if you wanted to add a crate to the build system you had to
change about 100 lines across 8 separate makefiles. This is highly error prone
and opaque to all but a few. This refactoring is targeted at consolidating this
effort so adding a new crate adds one line in one file in a way that everyone
can understand it.

Author: alexcrichton

Makefile.in

@@ -222,72 +222,27 @@ GENERATED :=
 
 
 ######################################################################
-# Crates
+# Cleaning out old crates
 ######################################################################
 
-define DEF_LIBS
-
-CFG_RUNTIME_$(1) :=$(call CFG_STATIC_LIB_NAME_$(1),rustrt)
-CFG_RUSTLLVM_$(1) :=$(call CFG_STATIC_LIB_NAME_$(1),rustllvm)
-CFG_STDLIB_$(1) :=$(call CFG_LIB_NAME_$(1),std)
-CFG_EXTRALIB_$(1) :=$(call CFG_LIB_NAME_$(1),extra)
-CFG_LIBRUSTC_$(1) :=$(call CFG_LIB_NAME_$(1),rustc)
-CFG_LIBSYNTAX_$(1) :=$(call CFG_LIB_NAME_$(1),syntax)
-CFG_LIBRUSTPKG_$(1) :=$(call CFG_LIB_NAME_$(1),rustpkg)
-CFG_LIBRUSTDOC_$(1) :=$(call CFG_LIB_NAME_$(1),rustdoc)
-CFG_LIBRUSTUV_$(1) :=$(call CFG_LIB_NAME_$(1),rustuv)
-CFG_LIBGREEN_$(1) :=$(call CFG_LIB_NAME_$(1),green)
-CFG_LIBNATIVE_$(1) :=$(call CFG_LIB_NAME_$(1),native)
-
-EXTRALIB_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),extra)
-STDLIB_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),std)
-LIBRUSTC_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),rustc)
-LIBSYNTAX_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),syntax)
-LIBRUSTPKG_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),rustpkg)
-LIBRUSTDOC_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),rustdoc)
-LIBRUSTUV_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),rustuv)
-LIBGREEN_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),green)
-LIBNATIVE_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),native)
-EXTRALIB_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),extra)
-STDLIB_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),std)
-LIBRUSTC_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),rustc)
-LIBSYNTAX_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),syntax)
-LIBRUSTPKG_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),rustpkg)
-LIBRUSTDOC_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),rustdoc)
-LIBRUSTUV_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),rustuv)
-LIBGREEN_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),green)
-LIBNATIVE_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),native)
-
-EXTRALIB_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,extra)
-STDLIB_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,std)
-LIBRUSTUV_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,rustuv)
-LIBSYNTAX_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,syntax)
-LIBRUSTC_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,rustc)
-LIBNATIVE_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,native)
-LIBGREEN_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,green)
-
-endef
-
 # $(1) is the path for directory to match against
 # $(2) is the glob to use in the match
-# $(3) is filename (usually the target being created) to filter out from match
-#      (i.e. filename is not out-of-date artifact from prior Rust version/build)
 #
 # Note that a common bug is to accidentally construct the glob denoted
 # by $(2) with a space character prefix, which invalidates the
 # construction $(1)$(2).
-define CHECK_FOR_OLD_GLOB_MATCHES_EXCEPT
-  $(Q)MATCHES="$(filter-out %$(3),$(wildcard $(1)/$(2)))"; if [ -n "$$MATCHES" ] ; then echo "warning: there are previous" \'$(2)\' "libraries:" $$MATCHES; fi
+define CHECK_FOR_OLD_GLOB_MATCHES
+  $(Q)MATCHES="$(wildcard $(1))"; if [ -n "$$MATCHES" ] ; then echo "warning: there are previous" \'$(notdir $(2))\' "libraries:" $$MATCHES; fi
 endef
 
 # Same interface as above, but deletes rather than just listing the files.
 ifdef VERBOSE
-define REMOVE_ALL_OLD_GLOB_MATCHES_EXCEPT
-  $(Q)MATCHES="$(filter-out %$(3),$(wildcard $(1)/$(2)))"; if [ -n "$$MATCHES" ] ; then echo "warning: removing previous" \'$(2)\' "libraries:" $$MATCHES; rm $$MATCHES ; fi
+define REMOVE_ALL_OLD_GLOB_MATCHES
+  $(Q)MATCHES="$(wildcard $(1))"; if [ -n "$$MATCHES" ] ; then echo "warning: removing previous" \'$(notdir $(1))\' "libraries:" $$MATCHES; rm $$MATCHES ; fi
 endef
 else
-define REMOVE_ALL_OLD_GLOB_MATCHES_EXCEPT
-  $(Q)MATCHES="$(filter-out %$(3),$(wildcard $(1)/$(2)))"; if [ -n "$$MATCHES" ] ; then rm $$MATCHES ; fi
+define REMOVE_ALL_OLD_GLOB_MATCHES
+  $(Q)MATCHES="$(wildcard $(1))"; if [ -n "$$MATCHES" ] ; then rm $$MATCHES ; fi
 endef
 endif
 
@@ -298,72 +253,15 @@ endif
 # soon. (This is in contrast to the macros above, which are meant to
 # be run at the outset of a command list in a rule.)
 ifdef VERBOSE
-define LIST_ALL_OLD_GLOB_MATCHES_EXCEPT
-  @echo "info: now are following matches for" '$(2)' "libraries:"
-  @( cd $(1) && ( ls $(2) 2>/dev/null || true ) | grep -v $(3) || true )
+define LIST_ALL_OLD_GLOB_MATCHES
+  @echo "info: now are following matches for" '$(notdir $(1))' "libraries:"
+  @( ls $(1) 2>/dev/null || true )
 endef
 else
-define LIST_ALL_OLD_GLOB_MATCHES_EXCEPT
+define LIST_ALL_OLD_GLOB_MATCHES
 endef
 endif
 
-$(foreach target,$(CFG_TARGET),\
-  $(eval $(call DEF_LIBS,$(target))))
-
-######################################################################
-# Standard library variables
-######################################################################
-
-STDLIB_CRATE := $(S)src/libstd/lib.rs
-STDLIB_INPUTS := $(wildcard $(addprefix $(S)src/libstd/,        \
-                                           *.rs */*.rs */*/*rs */*/*/*rs))
-
-######################################################################
-# Extra library variables
-######################################################################
-
-EXTRALIB_CRATE := $(S)src/libextra/lib.rs
-EXTRALIB_INPUTS := $(wildcard $(addprefix $(S)src/libextra/,          \
-                                          *.rs */*.rs))
-
-######################################################################
-# Rust UV library variables
-######################################################################
-
-LIBRUSTUV_CRATE := $(S)src/librustuv/lib.rs
-LIBRUSTUV_INPUTS := $(wildcard $(addprefix $(S)src/librustuv/,          \
-                                          *.rs */*.rs))
-
-######################################################################
-# Green threading library variables
-######################################################################
-
-LIBGREEN_CRATE := $(S)src/libgreen/lib.rs
-LIBGREEN_INPUTS := $(wildcard $(addprefix $(S)src/libgreen/,          \
-                                          *.rs */*.rs))
-
-######################################################################
-# Native threading library variables
-######################################################################
-
-LIBNATIVE_CRATE := $(S)src/libnative/lib.rs
-LIBNATIVE_INPUTS := $(wildcard $(addprefix $(S)src/libnative/,          \
-                                          *.rs */*.rs))
-
-######################################################################
-# rustc crate variables
-######################################################################
-
-COMPILER_CRATE := $(S)src/librustc/lib.rs
-COMPILER_INPUTS := $(wildcard $(addprefix $(S)src/librustc/,      \
-                           *.rs */*.rs */*/*.rs */*/*/*.rs))
-
-LIBSYNTAX_CRATE := $(S)src/libsyntax/lib.rs
-LIBSYNTAX_INPUTS := $(wildcard $(addprefix $(S)src/libsyntax/, \
-                           *.rs */*.rs */*/*.rs */*/*/*.rs))
-
-DRIVER_CRATE := $(S)src/driver/driver.rs
-
 ######################################################################
 # LLVM macros
 ######################################################################
@@ -424,14 +322,12 @@ export CFG_RUSTLIBDIR
 export CFG_LIBDIR_RELATIVE
 export CFG_DISABLE_INJECT_STD_VERSION
 
-######################################################################
-# Subprograms
-######################################################################
-
 ######################################################################
 # Per-stage targets and runner
 ######################################################################
 
+include $(CFG_SRC_DIR)mk/crates.mk
+
 define SREQ
 # $(1) is the stage number
 # $(2) is the target triple
@@ -447,84 +343,37 @@ TROOT$(1)_T_$(2)_H_$(3) = $$(HLIB$(1)_H_$(3))/$$(CFG_RUSTLIBDIR)/$(2)
 TBIN$(1)_T_$(2)_H_$(3) = $$(TROOT$(1)_T_$(2)_H_$(3))/bin
 TLIB$(1)_T_$(2)_H_$(3) = $$(TROOT$(1)_T_$(2)_H_$(3))/lib
 
-# The name of the standard and extra libraries used by rustc
-HSTDLIB_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_STDLIB_$(3))
-TSTDLIB_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_STDLIB_$(2))
-
-HEXTRALIB_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_EXTRALIB_$(3))
-TEXTRALIB_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_EXTRALIB_$(2))
-
-HLIBRUSTC_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_LIBRUSTC_$(3))
-TLIBRUSTC_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTC_$(2))
-
-HLIBRUSTUV_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_LIBRUSTUV_$(3))
-TLIBRUSTUV_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTUV_$(2))
-
-HLIBGREEN_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_LIBGREEN_$(3))
-TLIBGREEN_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBGREEN_$(2))
-
-HLIBNATIVE_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_LIBNATIVE_$(3))
-TLIBNATIVE_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBNATIVE_$(2))
-
 # Preqrequisites for using the stageN compiler
 ifeq ($(1),0)
 HSREQ$(1)_H_$(3) = $$(HBIN$(1)_H_$(3))/rustc$$(X_$(3))
 else
 HSREQ$(1)_H_$(3) = \
 	$$(HBIN$(1)_H_$(3))/rustc$$(X_$(3)) \
-	$$(HSTDLIB_DEFAULT$(1)_H_$(3)) \
-	$$(HEXTRALIB_DEFAULT$(1)_H_$(3)) \
-	$$(HLIBSYNTAX_DEFAULT$(1)_H_$(3)) \
-	$$(HLIBRUSTC_DEFAULT$(1)_H_$(3)) \
-	$$(HLIBRUSTUV_DEFAULT$(1)_H_$(3)) \
-	$$(HLIBGREEN_DEFAULT$(1)_H_$(3)) \
-	$$(HLIBNATIVE_DEFAULT$(1)_H_$(3)) \
+	$$(HLIB$(1)_H_$(3))/stamp.rustc \
+	$$(foreach dep,$$(RUST_DEPS_rustc),$$(HLIB$(1)_H_$(3))/stamp.$$(dep)) \
 	$$(MKFILE_DEPS)
 endif
 
 # Prerequisites for using the stageN compiler to build target artifacts
 TSREQ$(1)_T_$(2)_H_$(3) = \
 	$$(HSREQ$(1)_H_$(3)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_RUNTIME_$(2)) \
 	$$(TLIB$(1)_T_$(2)_H_$(3))/libmorestack.a
 
-# Prerequisites for a working stageN compiler and libraries, for a specific target
+# Prerequisites for a working stageN compiler and libraries, for a specific
+# target
 SREQ$(1)_T_$(2)_H_$(3) = \
 	$$(TSREQ$(1)_T_$(2)_H_$(3)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_STDLIB_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_EXTRALIB_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTUV_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBGREEN_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBNATIVE_$(2))
+	$$(foreach dep,$$(TARGET_CRATES),\
+	    $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(dep))
 
-# Prerequisites for a working stageN compiler and libraries, for a specific target
+# Prerequisites for a working stageN compiler and complete set of target
+# libraries
 CSREQ$(1)_T_$(2)_H_$(3) = \
 	$$(TSREQ$(1)_T_$(2)_H_$(3)) \
 	$$(HBIN$(1)_H_$(3))/rustpkg$$(X_$(3)) \
 	$$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
-	$$(HLIB$(1)_H_$(3))/$(CFG_LIBRUSTPKG_$(3)) \
-	$$(HLIB$(1)_H_$(3))/$(CFG_LIBRUSTDOC_$(3)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_STDLIB_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_EXTRALIB_$(2))  \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBSYNTAX_$(2))  \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTC_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTPKG_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTDOC_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTUV_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBGREEN_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBNATIVE_$(2))
+	$$(foreach dep,$$(CRATES),$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(dep)) \
+	$$(foreach dep,$$(HOST_CRATES),$$(HLIB$(1)_H_$(3))/stamp.$$(dep))
 
 ifeq ($(1),0)
 # Don't run the the stage0 compiler under valgrind - that ship has sailed
@@ -562,15 +411,15 @@ endif
 STAGE$(1)_T_$(2)_H_$(3) := 						\
 	$$(Q)$$(RPATH_VAR$(1)_T_$(2)_H_$(3))                            \
 		$$(call CFG_RUN_TARG_$(3),$(1),				\
-		$$(CFG_VALGRIND_COMPILE$(1)) 			\
+		$$(CFG_VALGRIND_COMPILE$(1))				\
 		$$(HBIN$(1)_H_$(3))/rustc$$(X_$(3))			\
 		--cfg $$(CFGFLAG$(1)_T_$(2)_H_$(3))			\
 		$$(CFG_RUSTC_FLAGS) $$(EXTRAFLAGS_STAGE$(1)) --target=$(2)) \
                 $$(RUSTC_FLAGS_$(2))
 
-PERF_STAGE$(1)_T_$(2)_H_$(3) := 				\
+PERF_STAGE$(1)_T_$(2)_H_$(3) :=						\
 	$$(Q)$$(call CFG_RUN_TARG_$(3),$(1),				\
-		$$(CFG_PERF_TOOL) 						\
+		$$(CFG_PERF_TOOL) 					\
 		$$(HBIN$(1)_H_$(3))/rustc$$(X_$(3))			\
 		--cfg $$(CFGFLAG$(1)_T_$(2)_H_$(3))			\
 		$$(CFG_RUSTC_FLAGS) $$(EXTRAFLAGS_STAGE$(1)) --target=$(2)) \
@@ -594,13 +443,12 @@ define DEF_RUSTC_STAGE_TARGET
 # $(2) == stage
 
 rustc-stage$(2)-H-$(1):							\
-	$$(foreach target,$$(CFG_TARGET),	\
-		$$(SREQ$(2)_T_$$(target)_H_$(1)))
+	$$(foreach target,$$(CFG_TARGET),$$(SREQ$(2)_T_$$(target)_H_$(1)))
 
 endef
 
-$(foreach host,$(CFG_HOST),							\
- $(eval $(foreach stage,1 2 3,									\
+$(foreach host,$(CFG_HOST),						\
+ $(eval $(foreach stage,1 2 3,						\
   $(eval $(call DEF_RUSTC_STAGE_TARGET,$(host),$(stage))))))
 
 rustc-stage1: rustc-stage1-H-$(CFG_BUILD)
@@ -701,7 +549,6 @@ include $(CFG_SRC_DIR)mk/target.mk
 include $(CFG_SRC_DIR)mk/host.mk
 include $(CFG_SRC_DIR)mk/stage0.mk
 include $(CFG_SRC_DIR)mk/rustllvm.mk
-include $(CFG_SRC_DIR)mk/tools.mk
 include $(CFG_SRC_DIR)mk/docs.mk
 include $(CFG_SRC_DIR)mk/llvm.mk