depends: add shared dependency builder

See the README's in depends for documentation
TrezarCoin · Aug 8, 2014 · 1dec09b · 1dec09b
1 parent 36065cc
commit 1dec09b
Show file tree

Hide file tree

Showing 49 changed files with 4,992 additions and 0 deletions.
diff --git a/depends/.gitignore b/depends/.gitignore
@@ -0,0 +1,5 @@
+SDKs/
+work/
+built/
+sources/
+config.site
diff --git a/depends/Makefile b/depends/Makefile
@@ -0,0 +1,114 @@
+.NOTPARALLEL :
+
+SOURCES_PATH ?= $(BASEDIR)/sources
+BASE_CACHE ?= $(BASEDIR)/built
+SDK_PATH ?= $(BASEDIR)/SDKs
+NO_QT ?=
+NO_WALLET ?=
+NO_UPNP ?=
+
+BUILD = $(shell ./config.guess)
+HOST ?= $(BUILD)
+PATCHES_PATH = $(BASEDIR)/patches
+BASEDIR = $(CURDIR)
+HASH_LENGTH:=11
+
+host:=$(BUILD)
+ifneq ($(HOST),)
+host:=$(HOST)
+host_toolchain:=$(HOST)-
+endif
+
+base_build_dir=$(BASEDIR)/work/build
+base_staging_dir=$(BASEDIR)/work/staging
+canonical_host:=$(shell ./config.sub $(HOST))
+build:=$(shell ./config.sub $(BUILD))
+
+build_arch =$(firstword $(subst -, ,$(build)))
+build_vendor=$(word 2,$(subst -, ,$(build)))
+full_build_os:=$(subst $(build_arch)-$(build_vendor)-,,$(build))
+build_os:=$(findstring linux,$(full_build_os))
+build_os+=$(findstring darwin,$(full_build_os))
+build_os:=$(strip $(build_os))
+ifeq ($(build_os),)
+build_os=$(full_build_os)
+endif
+
+host_arch=$(firstword $(subst -, ,$(canonical_host)))
+host_vendor=$(word 2,$(subst -, ,$(canonical_host)))
+full_host_os:=$(subst $(host_arch)-$(host_vendor)-,,$(canonical_host))
+host_os:=$(findstring linux,$(full_host_os))
+host_os+=$(findstring darwin,$(full_host_os))
+host_os+=$(findstring mingw32,$(full_host_os))
+host_os:=$(strip $(host_os))
+ifeq ($(host_os),)
+host_os=$(full_host_os)
+endif
+
+$(host_arch)_$(host_os)_prefix=$(BASEDIR)/$(host)
+$(host_arch)_$(host_os)_host=$(host)
+host_prefix=$($(host_arch)_$(host_os)_prefix)
+build_prefix=$(host_prefix)/native
+build_host=$(build)
+
+AT_$(V):=
+AT_:=@
+AT:=$(AT_$(V))
+
+all: install
+
+include hosts/$(host_os).mk
+include hosts/default.mk
+include builders/$(build_os).mk
+include builders/default.mk
+include packages/packages.mk
+
+qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages)
+qt_native_packages_$(NO_QT) = $(qt_native_packages)
+wallet_packages_$(NO_WALLET) = $(wallet_packages)
+upnp_packages_$(NO_UPNP) = $(upnp_packages)
+
+packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_)
+native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) $(qt_native_packages_)
+all_packages = $(packages) $(native_packages)
+
+meta_depends = Makefile builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
+
+$(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
+
+include funcs.mk
+
+toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
+final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)
+final_build_id+=$(shell echo -n $(final_build_id_long) | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
+$(host_prefix)/.stamp_$(final_build_id): | $(native_packages) $(packages)
+	$(AT)rm -rf $(@D)
+	$(AT)mkdir -p $(@D)
+	$(AT)echo copying packages: $|
+	$(AT)echo to: $(@D)
+	$(AT)cd $(@D); $(foreach package,$|, tar xf $($(package)_cached); )
+	$(AT)touch $@
+
+$(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id)
+	$(AT)@mkdir -p $(@D)
+	$(AT)sed -e 's|@HOST@|$(host)|' \
+            -e 's|@CC@|$(toolchain_path)$(host_CC)|' \
+            -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \
+            -e 's|@AR@|$(toolchain_path)$(host_AR)|' \
+            -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \
+            -e 's|@NM@|$(toolchain_path)$(host_NM)|' \
+            -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \
+            -e 's|@build_os@|$(build_os)|' \
+            -e 's|@host_os@|$(host_os)|' \
+            -e 's|@CFLAGS@|$(host_CFLAGS)|' \
+            -e 's|@CXXFLAGS@|$(host_CXXFLAGS)|' \
+            -e 's|@LDFLAGS@|$(host_LDFLAGS)|' \
+            -e 's|@no_qt@|$(NO_QT)|' \
+            -e 's|@no_wallet@|$(NO_WALLET)|' \
+            -e 's|@no_upnp@|$(NO_UPNP)|' \
+            $< > $@
+	$(AT)touch $@
+
+install: $(host_prefix)/share/config.site
+download: $(all_sources)
+.PHONY: install cached
diff --git a/depends/README b/depends/README
@@ -0,0 +1,55 @@
+This is a system of building and caching dependencies necessary for building
+Bitcoin.
+
+There are several features that make it different from most similar systems:
+
+- It is designed to be builder and host agnostic
+
+In theory, binaries for any target OS/architecture can be created, from a
+builder running any OS/architecture. In practice, build-side tools must be
+specified when the defaults don't fit, and packages must be ammended to work
+on new hosts. For now, a build architecture of x86_64 is assumed, either on
+Linux or OSX.
+
+- No reliance on timestamps
+
+File presence is used to determine what needs to be built. This makes the
+results distributable and easily digestable by automated builders.
+
+- Each build only has its specified dependencies available at build-time.
+
+For each build, the sysroot is wiped and the (recursive) dependencies are
+installed. This makes each build deterministic, since there will never be any
+unknown files available to cause side-effects.
+
+- Each package is cached and only rebuilt as needed.
+
+Before building, a unique build-id is generated for each package. This id
+consists of a hash of all files used to build the package (Makefiles, packages,
+etc), and as well as a hash of the same data for each recursive dependency. If
+any portion of a package's build recipe changes, it will be rebuilt as well as
+any other package that depends on it. If any of the main makefiles (Makefile, 
+funcs.mk, etc) are changed, all packages will be rebuilt. After building, the
+results are cached into a tarball that can be re-used and distributed.
+
+- Package build results are (relatively) deterministic.
+
+Each package is configured and patched so that it will yield the same
+build-results with each consequent build, within a reasonable set of
+constraints. Some things like timestamp insertion are unavoidable, and are
+beyond the scope of this system. Additionally, the toolchain itself must be
+capable of deterministic results. When revisions are properly bumped, a cached
+build should represent an exact single payload.
+
+- Sources are fetched and verified automatically
+
+Each package must define its source location and checksum. The build will fail
+if the fetched source does not match. Sources may be pre-seeded and/or cached
+as desired.
+
+- Self-cleaning
+
+Build and staging dirs are wiped after use, and any previous version of a
+cached result is removed following a successful build. Automated builders
+should be able to build each revision and store the results with no further
+intervention.
diff --git a/depends/README.packages b/depends/README.packages
@@ -0,0 +1,116 @@
+Each recipe consists of 3 main parts: defining identifiers, setting build
+variables, and defining build commands.
+
+The package "mylib" will be used here as an example
+
+General tips:
+mylib_foo is written as $(package)_foo in order to make recipes more similar.
+
+Identifiers:
+Each package is required to define at least these variables:
+  $(package)_version:
+    Version of the upstream library or program. If there is no version, a
+    placeholder such as 1.0 can be used.
+  $(package)_download_path:
+    Location of the upstream source, without the file-name. Usually http or
+    ftp.
+  $(package)_file_name:
+    The upstream source filename available at the download path.
+  $(package)_sha256_hash:
+    The sha256 hash of the upstream file
+
+These variables are optional:
+  $(package)_build_subdir:
+    cd to this dir before running configure/build/stage commands.
+  $(package)_download_file:
+    The file-name of the upstream source if it differs from how it should be
+    stored locally. This can be used to avoid storing file-names with strange
+    characters.
+  $(package)_dependencies:
+    Names of any other packages that this one depends on.
+  $(package)_patches:
+    Filenames of any patches needed to build the package
+
+
+Build Variables:
+After defining the main identifiers, build variables may be added or customized
+before running the build commands. They should be added to a function called
+$(package)_set_vars. For example:
+
+define $(package)_set_vars
+...
+endef
+
+Most variables can be prefixed with the host, architecture, or both, to make
+the modifications specific to that case. For example:
+
+  Universal:         $(package)_cc=gcc
+  Linux only:        $(package)_linux_cc=gcc
+  x86_64 only:       $(package)_x86_64_cc = gcc
+  x86_64 linux only: $(package)_x86_64_linux_cc = gcc
+
+These variables may be set to override or append their default values.
+  $(package)_cc
+  $(package)_cxx
+  $(package)_objc
+  $(package)_objcxx
+  $(package)_ar
+  $(package)_ranlib
+  $(package)_libtool
+  $(package)_nm
+  $(package)_cflags
+  $(package)_cxxflags
+  $(package)_ldflags
+  $(package)_cppflags
+  $(package)_config_env
+  $(package)_build_env
+  $(package)_stage_env
+
+The *_env variables are used to add environment variables to the respective
+commands.
+
+Other variables may be defined as needed.
+
+Build commands:
+
+  For each build, a unique build dir and staging dir are created. For example,
+  work/build/mylib/1.0-1adac830f6e and work/staging/mylib/1.0-1adac830f6e.
+
+  The following build commands are available for each recipe:
+
+  $(package)_fetch_cmds:
+    Runs from: build dir
+    Fetch the source file. If undefined, it will be fetched and verified
+    against its hash.
+  $(package)_extract_cmds:
+    Runs from: build dir
+    Verify the source file against its hash and extract it. If undefined, the
+    source is assumed to be a tarball.
+  $(package)_preprocess_cmds:
+    Runs from: build dir/$(package)_build_subdir
+    Preprocess the source as necessary. If undefined, does nothing.
+  $(package)_config_cmds:
+    Runs from: build dir/$(package)_build_subdir
+    Configure the source. If undefined, does nothing.
+  $(package)_build_cmds:
+    Runs from: build dir/$(package)_build_subdir
+    Build the source. If undefined, does nothing.
+  $(package)_stage_cmds:
+    Runs from: build dir/$(package)_build_subdir
+    Stage the build results. If undefined, does nothing.
+
+  The following variables are available for each recipe:
+  $(1)_staging_dir: package's destination sysroot path
+  $(1)_staging_prefix_dir: prefix path inside of the package's staging dir
+  $(1)_extract_dir: path to the package's extracted sources
+  $(1)_build_dir: path where configure/build/stage commands will be run
+  $(1)_patch_dir: path where the package's patches (if any) are found
+
+Notes on build commands:
+
+For packages built with autotools, $($(package)_autoconf) can be used in the
+configure step to (usually) correctly configure automatically. Any
+$($(package)_config_opts) will be appended.
+
+Most autotools projects can be properly staged using:
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
diff --git a/depends/README.usage b/depends/README.usage
@@ -0,0 +1,29 @@
+To build dependencies for the current arch+OS:
+    make
+To build for another arch/OS:
+  make HOST=host-platform-triplet && make HOST=host-platform-triplet
+  (For example: make HOST=i686-w64-mingw32 -j4)
+
+A prefix will be generated that's suitable for plugging into Bitcoin's
+configure. In the above example, a dir named i686-w64-mingw32 will be
+created. To use it for Bitcoin:
+
+./configure --prefix=`pwd`/depends/i686-w64-mingw32
+
+No other options are needed, the paths are automatically configured.
+
+Dependency Options:
+The following can be set when running make: make FOO=bar
+
+SOURCES_PATH: downloaded sources will be placed here
+BASE_CACHE: built packages will be placed here
+SDK_PATH: Path where sdk's can be found (used by OSX)
+NO_QT: Don't download/build/cache qt and its dependencies
+NO_WALLET: Don't download/build/cache libs needed to enable the wallet
+NO_UPNP: Don't download/build/cache packages needed for enabling upnp
+
+If some packages are not built, for example 'make NO_WALLET=1', the appropriate
+options will be passed to bitcoin's configure. In this case, --disable-wallet.
+
+Additional targets:
+download: run 'make download' to fetch sources without building them
diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk
@@ -0,0 +1,22 @@
+build_darwin_CC: = $(shell xcrun -f clang)
+build_darwin_CXX: = $(shell xcrun -f clang++)
+build_darwin_AR: = $(shell xcrun -f ar)
+build_darwin_RANLIB: = $(shell xcrun -f ranlib)
+build_darwin_STRIP: = $(shell xcrun -f strip)
+build_darwin_OTOOL: = $(shell xcrun -f otool)
+build_darwin_NM: = $(shell xcrun -f nm)
+build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
+build_darwin_SHA256SUM = shasum -a 256
+build_darwin_DOWNLOAD = curl -L -o
+
+#darwin host on darwin builder. overrides darwin host preferences.
+darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION)
+darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION)
+darwin_AR:=$(shell xcrun -f ar)
+darwin_RANLIB:=$(shell xcrun -f ranlib)
+darwin_STRIP:=$(shell xcrun -f strip)
+darwin_LIBTOOL:=$(shell xcrun -f libtool)
+darwin_OTOOL:=$(shell xcrun -f otool)
+darwin_NM:=$(shell xcrun -f nm)
+darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
+darwin_native_toolchain=
diff --git a/depends/builders/default.mk b/depends/builders/default.mk
@@ -0,0 +1,20 @@
+default_build_CC = gcc
+default_build_CXX = g++
+default_build_AR = ar
+default_build_RANLIB = ranlib
+default_build_STRIP = strip
+default_build_NM = nm
+default_build_OTOOL = otool
+default_build_INSTALL_NAME_TOOL = install_name_tool
+
+define add_build_tool_func
+build_$(build_os)_$1 ?= $$(default_build_$1)
+build_$(build_arch)_$(build_os)_$1 ?= $$(build_$(build_os)_$1)
+build_$1=$$(build_$(build_arch)_$(build_os)_$1)
+endef
+$(foreach var,CC CXX AR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL,$(eval $(call add_build_tool_func,$(var))))
+define add_build_flags_func
+build_$(build_arch)_$(build_os)_$1 += $(build_$(build_os)_$1)
+build_$1=$$(build_$(build_arch)_$(build_os)_$1)
+endef
+$(foreach flags, CFLAGS CXXFLAGS LDFLAGS, $(eval $(call add_build_flags_func,$(flags))))
diff --git a/depends/builders/linux.mk b/depends/builders/linux.mk
@@ -0,0 +1,2 @@
+build_linux_SHA256SUM = sha256sum
+build_linux_DOWNLOAD = wget -nv -O
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		build_linux_SHA256SUM = sha256sum
		build_linux_DOWNLOAD = wget -nv -O