From: Stefan Bund Date: Fri, 23 Jul 2010 13:14:37 +0000 (+0200) Subject: caching and offline initialization (init-nonet) X-Git-Url: http://g0dil.de/git?a=commitdiff_plain;h=8060e8fb933dacfe90cfb83ab2a4f54ca444aef9;p=zope-bootstrap.git caching and offline initialization (init-nonet) --- diff --git a/Makefile.master b/Makefile.master index 0e06fd1..4931d11 100644 --- a/Makefile.master +++ b/Makefile.master @@ -49,9 +49,80 @@ BASEDIR=$(shell pwd) +DLCACHE ?= dlcache + default: update-nonet .PHONY: default +# Usage: +# @$(httpget) [] +# +# Download to . If is missing, defaults to +# the last component of the . +# +# If the file exists in $(DLCACHE), it is retrieved from there instead. +define httpget + _httpget() { \ + name="$${1##*/}"; \ + target="$$2"; \ + [ -n "$$target" ] || target="$$name"; \ + cache="$(DLCACHE)/$$name"; \ + if [ -r "$$cache" ]; then \ + echo "Fetching '$$name' from cache."; \ + else \ + [ -z "$(NONET)" ] || ( echo "Missing '$$1'."; exit 2 ); \ + echo "Downloading '$$1'."; \ + wget -O "$$cache" "$$1"; \ + fi; \ + cp "$$cache" "$$target"; \ + }; \ + _httpget +endef + +# Usage: +# @$(dlegg) +# +# Downloads egg to '$(DLCACHE)/-*.egg' unless a file with that name +# already exists. +# +# THIS DOES NOT WORK WITH ALL PACKAGES !!! USE WITH CARE +define _dlegg + _dlegg() { \ + if [ -r $(DLCACHE)/$$1-*.egg ]; then \ + name="`cd $(DLCACHE); echo $$1-*.egg`"; \ + echo "Fetching '$$name' from cache."; \ + else \ + [ -z "$(NONET)" ] || ( echo "Missing '$$1'."; exit 2 ); \ + $(EASY_INSTALL) -zmaxd $(DLCACHE) $$1; \ + name="`cd $(DLCACHE); echo $$1-*.egg`"; \ + if [ -d $(DLCACHE)/$$name ]; then \ + cd $(DLCACHE)/$$name; \ + zip -r ../$$name.zip .; \ + cd ..; \ + rm -r $$name; \ + mv $$name.zip $$name; \ + fi; \ + fi; \ + } +endef +define dlegg + $(_dlegg); _dlegg +endef + +# Usage: +# @$(install) +# +# Downloads egg to '$(DLCACHE)/-*.egg' and installs it unless a file +# with that name already exists. +define install + _install() { \ + echo $(PIP_CACHE_FILES); \ + echo "$(PIP) install --download-cache="$(DLCACHE)/pip" $(PIP_OPTS) $$1"; \ + $(PIP) install --download-cache="$(DLCACHE)/pip" $(PIP_OPTS) "$$1"; \ + }; \ + _install +endef + ########################################################################### # Customization targets @@ -59,6 +130,7 @@ debdepends:: .PHONY: debdepends setup:: + mkdir -p $(DLCACHE) $(DLCACHE)/pip .PHONY: setup eggs:: @@ -75,7 +147,6 @@ update-hook:: bootstrap:: sed -i -e '1s/^#!.*\/python/#!$(subst /,\/,$(PYTHON))/' $(PYTHON_DIR)/bin/* -# $(PYTHON) bootstrap.py $(PYTHON) -c 'from zc.buildout.buildout import main; main(["bootstrap"])' update:: update-hook @@ -89,10 +160,11 @@ define gitignore _gitignore endef .gitignore:: - @echo "Updating .gitignore" + @echo "Updating .gitignore." @$(gitignore) "*.pyc" @$(gitignore) "*.egg-info/" @$(gitignore) "/.env" + @$(gitignore) "/$(DLCACHE)/" define env _env () { \ @@ -106,7 +178,7 @@ define env _env endef .env:: - @echo "Updating .env" + @echo "Updating .env." clean:: rm -f .gitignore .env @@ -115,15 +187,35 @@ clean:: ########################################################################### # internal targets +prepare-pipcache: + @( \ + cd $(DLCACHE)/pip; \ + for file in *.tar.gz; do \ + name="$${file##*%2F}"; \ + if [ "$$name" != "$$file" -a ! -r "$$name" ]; then \ + ln -s $$file $$name || exit 1; \ + fi; \ + done \ + ) + +PIP_CACHE_FILES := $(shell cd $(DLCACHE)/pip; ls *.tar.gz | grep -vF '%2F') + +do-init-nonet: PIP_OPTS = --no-index $(patsubst %,--find-link=file://$(BASEDIR)/$(DLCACHE)/pip/%,$(PIP_CACHE_FILES)) +do-init-nonet: BUILDOUT_OPTS = -N +do-init-nonet: NONET=1 +do-init-nonet: init ########################################################################### # user targets init: setup eggs buildout init-hook update-hook bootstrap update .gitignore .env +init-nonet: prepare-pipcache + @$(MAKE) do-init-nonet + versions: @echo "# Add the following lines to [versions] in buildout.cfg to pin all packages" - @bin/buildout -Novvvvv | sed -ne 's/^Picked: //p' | sort | uniq + @bin/buildout -vvvvv | sed -ne 's/^Picked: //p' | sort | uniq shell: @eval "`cat .env`"; $$SHELL diff --git a/Makefile.pil b/Makefile.pil index 7dbcf45..a525436 100644 --- a/Makefile.pil +++ b/Makefile.pil @@ -7,7 +7,7 @@ PIL_URL ?= http://effbot.org/downloads/Imaging-$(PIL_VERSION).tar.gz pil-unpack: $(PYTHON_DIR)/Extensions/Imaging-$(PIL_VERSION)/README $(PYTHON_DIR)/Extensions/Imaging-$(PIL_VERSION)/README: - wget $(PIL_URL) -O $(PYTHON_DIR)/Extensions/pil.tgz + @$(httpget) $(PIL_URL) $(PYTHON_DIR)/Extensions/pil.tgz tar -C $(PYTHON_DIR)/Extensions -xzf $(PYTHON_DIR)/Extensions/pil.tgz rm -f $(PYTHON_DIR)/Extensions/pil.tgz .PHONY: pil-unpack diff --git a/Makefile.plone3-buildout b/Makefile.plone3-buildout index afca669..520aa48 100644 --- a/Makefile.plone3-buildout +++ b/Makefile.plone3-buildout @@ -6,7 +6,7 @@ buildout.cfg: ########################################################################### eggs:: - $(EASY_INSTALL) ZopeSkel + @$(install) ZopeSkel buildout:: buildout.cfg diff --git a/Makefile.pybase b/Makefile.pybase index af6dc8a..b9db381 100644 --- a/Makefile.pybase +++ b/Makefile.pybase @@ -6,18 +6,19 @@ SETUPTOOLS_URL ?= http://peak.telecommunity.com/dist/ez_setup.py ########################################################################### -PYTHON_DIR=python +PYTHON_DIR ?= python PYTHON = $(BASEDIR)/$(PYTHON_DIR)/bin/python EASY_INSTALL = $(BASEDIR)/$(PYTHON_DIR)/bin/easy_install +PIP = $(BASEDIR)/$(PYTHON_DIR)/bin/pip PASTER = $(BASEDIR)/$(PYTHON_DIR)/bin/paster ########################################################################### python-unpack: $(PYTHON_DIR)/Python-$(PYTHON_VERSION)/README $(PYTHON_DIR)/Python-$(PYTHON_VERSION)/README: - mkdir $(PYTHON_DIR) - wget "$(PYTHON_URL)" -O $(PYTHON_DIR)/python.tgz + mkdir -p $(PYTHON_DIR) + @$(httpget) $(PYTHON_URL) $(PYTHON_DIR)/python.tgz tar -C $(PYTHON_DIR) -xzf $(PYTHON_DIR)/python.tgz rm -f $(PYTHON_DIR)/python.tgz .PHONY: python-unpack @@ -40,14 +41,30 @@ python-clean: python: python-unpack python-build .PHONY: python -setuptools: $(EASY_INSTALL) -$(EASY_INSTALL): - mkdir $(PYTHON_DIR)/Extensions - wget $(SETUPTOOLS_URL) -O $(PYTHON_DIR)/Extensions/ez_setup.py +setuptools: $(BASEDIR)/$(PYTHON_DIR)/bin/easy_install +$(BASEDIR)/$(PYTHON_DIR)/bin/easy_install: + mkdir -p $(PYTHON_DIR)/Extensions + @$(httpget) $(SETUPTOOLS_URL) $(PYTHON_DIR)/Extensions/ez_setup.py + @( \ + if [ -r $(DLCACHE)/setuptools-*.egg ]; then \ + name="`cd $(DLCACHE); echo setuptools-*.egg`"; \ + echo "Fetching '$$name' from cache."; \ + cp "$(DLCACHE)/$$name" $(PYTHON_DIR)/Extensions; \ + else \ + echo "Downloading setuptools."; \ + ( cd $(PYTHON_DIR)/Extensions && $(PYTHON) -c 'import ez_setup; ez_setup.download_setuptools()' ); \ + cp $(PYTHON_DIR)/Extensions/setuptools-*.egg $(DLCACHE); \ + fi; \ + ) cd $(PYTHON_DIR)/Extensions && $(PYTHON) ez_setup.py .PHONY: setuptools -pybase: python setuptools +pip: $(PIP) +$(PIP): + @$(dlegg) pip + $(EASY_INSTALL) $(DLCACHE)/pip-*.egg + +pybase: python setuptools pip .PHONY: pybase ########################################################################### @@ -56,7 +73,7 @@ debdepends:: aptitude install build-essential zlibc eggs:: - $(EASY_INSTALL) zc.buildout + @$(install) zc.buildout .gitignore:: @$(gitignore) /python/ diff --git a/Makefile.zope3-buildout b/Makefile.zope3-buildout index 5f6edb7..f79d198 100644 --- a/Makefile.zope3-buildout +++ b/Makefile.zope3-buildout @@ -13,14 +13,20 @@ buildout.cfg: echo "zopeproject --no-buildout \"$(PROJECTNAME)\"" && \ (echo "$(ADMINUSER)"; echo "$(ADMINPWD)"; echo "$(EGGDIR)") | $(ZOPEPROJECT) --no-buildout "$(PROJECTNAME)" && \ mv $(PROJECTNAME)/* $(BASEDIR)/ \ - ) || ( rm -f temp-project; false ) - @(rm -rf $(BASEDIR)/temp-project) + ) || ( rm -rf temp-project; false ) @echo + @(rm -rf $(BASEDIR)/temp-project) + @sed -i -e '/\[buildout\]/ba' -eb -e:a \ + -ea\\ -e 'download-cache = $(DLCACHE)/downloads' \ + -ea\\ -e 'install-from-cache = true' \ + -ea\\ -e 'extends-cache = $(DLCACHE)/extends' \ + buildout.cfg || ( rm -f buildout.cfg; false ) + @mkdir -p $(DLCACHE)/downloads $(DLCACHE)/extends || ( rm -f buildout.cfg; false ) ########################################################################### eggs:: - $(EASY_INSTALL) zopeproject + @$(install) zopeproject buildout:: buildout.cfg diff --git a/skel-zope3/Makefile b/skel-zope3/Makefile index 70060c2..f718315 100644 --- a/skel-zope3/Makefile +++ b/skel-zope3/Makefile @@ -1,10 +1,10 @@ # -*- makefile -*- -PROJECTNAME := myproject +PROJECTNAME := appmain ########################################################################### -MAKELIB := $(shell for lib in plone-skeleton . ..; do [ ! -r "$$lib"/Makefile.master ] || break; done; echo "`cd $$lib; pwd`" ) +MAKELIB := $(shell for lib in plone-skeleton . ..; do [ ! -r "$$lib"/Makefile.master ] || break; done; echo "`cd $$lib; pwd`") include $(MAKELIB)/Makefile.master include $(MAKELIB)/Makefile.pybase