caching and offline initialization (init-nonet)
Stefan Bund [Fri, 23 Jul 2010 13:14:37 +0000 (15:14 +0200)]
Makefile.master
Makefile.pil
Makefile.plone3-buildout
Makefile.pybase
Makefile.zope3-buildout
skel-zope3/Makefile

index 0e06fd1..4931d11 100644 (file)
 
 BASEDIR=$(shell pwd)
 
+DLCACHE ?= dlcache
+
 default: update-nonet
 .PHONY: default
 
+# Usage:
+#     @$(httpget) <url> [<target>]
+#
+# Download <url> to <target>. If <target> is missing, <target> defaults to
+# the last component of the <url>.
+#
+# If the file exists in $(DLCACHE), it is retrieved from there instead.
+define httpget
+    _httpget() {                                                       \
+        name="$${1##*/}";                                              \
+       target="$$2";                                                   \
+       [ -n "$$target" ] || target="$$name";                           \
+        cache="$(DLCACHE)/$$name";                                     \
+       if [ -r "$$cache" ]; then                                       \
+           echo "Fetching '$$name' from cache.";                       \
+       else                                                            \
+           [ -z "$(NONET)" ] || ( echo "Missing '$$1'."; exit 2 );     \
+           echo "Downloading '$$1'.";                                  \
+           wget -O "$$cache" "$$1";                                    \
+       fi;                                                             \
+       cp "$$cache" "$$target";                                        \
+    };                                                                 \
+    _httpget
+endef
+
+# Usage:
+#     @$(dlegg) <package>
+#
+# Downloads egg to '$(DLCACHE)/<package>-*.egg' unless a file with that name
+# already exists.
+#
+# THIS DOES NOT WORK WITH ALL PACKAGES !!! USE WITH CARE
+define _dlegg
+    _dlegg() {                                                         \
+       if [ -r $(DLCACHE)/$$1-*.egg ]; then                            \
+           name="`cd $(DLCACHE); echo $$1-*.egg`";                     \
+           echo "Fetching '$$name' from cache.";                       \
+       else                                                            \
+           [ -z "$(NONET)" ] || ( echo "Missing '$$1'."; exit 2 );     \
+           $(EASY_INSTALL) -zmaxd $(DLCACHE) $$1;                      \
+           name="`cd $(DLCACHE); echo $$1-*.egg`";                     \
+           if [ -d $(DLCACHE)/$$name ]; then                           \
+               cd $(DLCACHE)/$$name;                                   \
+               zip -r ../$$name.zip .;                                 \
+               cd ..;                                                  \
+               rm -r $$name;                                           \
+               mv $$name.zip $$name;                                   \
+           fi;                                                         \
+       fi;                                                             \
+    }
+endef
+define dlegg
+    $(_dlegg); _dlegg
+endef
+
+# Usage:
+#     @$(install) <package>
+#
+# Downloads egg to '$(DLCACHE)/<package>-*.egg' and installs it unless a file
+# with that name already exists.
+define install
+    _install() { \
+       echo $(PIP_CACHE_FILES); \
+       echo "$(PIP) install --download-cache="$(DLCACHE)/pip" $(PIP_OPTS) $$1";        \
+       $(PIP) install --download-cache="$(DLCACHE)/pip" $(PIP_OPTS) "$$1";             \
+    };                                                                                 \
+    _install
+endef
+
 ###########################################################################
 # Customization targets
 
@@ -59,6 +130,7 @@ debdepends::
 .PHONY: debdepends
 
 setup::
+       mkdir -p $(DLCACHE) $(DLCACHE)/pip
 .PHONY: setup
 
 eggs::
@@ -75,7 +147,6 @@ update-hook::
 
 bootstrap::
        sed -i -e '1s/^#!.*\/python/#!$(subst /,\/,$(PYTHON))/' $(PYTHON_DIR)/bin/*
-#       $(PYTHON) bootstrap.py
        $(PYTHON) -c 'from zc.buildout.buildout import main; main(["bootstrap"])'
 
 update:: update-hook
@@ -89,10 +160,11 @@ define gitignore
     _gitignore
 endef
 .gitignore::
-       @echo "Updating .gitignore"
+       @echo "Updating .gitignore."
        @$(gitignore) "*.pyc"
        @$(gitignore) "*.egg-info/"
        @$(gitignore) "/.env"
+       @$(gitignore) "/$(DLCACHE)/"
 
 define env
     _env () {                                                  \
@@ -106,7 +178,7 @@ define env
     _env
 endef
 .env::
-       @echo "Updating .env"
+       @echo "Updating .env."
 
 clean::
        rm -f .gitignore .env
@@ -115,15 +187,35 @@ clean::
 ###########################################################################
 # internal targets
 
+prepare-pipcache:
+       @(                                                              \
+           cd $(DLCACHE)/pip;                                          \
+           for file in *.tar.gz; do                                    \
+               name="$${file##*%2F}";                                  \
+               if [ "$$name" != "$$file" -a ! -r "$$name" ]; then      \
+                   ln -s $$file $$name || exit 1;                      \
+               fi;                                                     \
+           done                                                        \
+       )
+
+PIP_CACHE_FILES := $(shell cd $(DLCACHE)/pip; ls *.tar.gz | grep -vF '%2F')
+
+do-init-nonet: PIP_OPTS  = --no-index $(patsubst %,--find-link=file://$(BASEDIR)/$(DLCACHE)/pip/%,$(PIP_CACHE_FILES))
+do-init-nonet: BUILDOUT_OPTS = -N
+do-init-nonet: NONET=1
+do-init-nonet: init
 
 ###########################################################################
 # user targets
 
 init: setup eggs buildout init-hook update-hook bootstrap update .gitignore .env
 
+init-nonet: prepare-pipcache
+       @$(MAKE) do-init-nonet
+
 versions:
        @echo "# Add the following lines to [versions] in buildout.cfg to pin all packages"
-       @bin/buildout -Novvvvv | sed -ne 's/^Picked: //p' | sort | uniq 
+       @bin/buildout -vvvvv | sed -ne 's/^Picked: //p' | sort | uniq 
 
 shell:
        @eval "`cat .env`"; $$SHELL
index 7dbcf45..a525436 100644 (file)
@@ -7,7 +7,7 @@ PIL_URL         ?= http://effbot.org/downloads/Imaging-$(PIL_VERSION).tar.gz
 
 pil-unpack: $(PYTHON_DIR)/Extensions/Imaging-$(PIL_VERSION)/README
 $(PYTHON_DIR)/Extensions/Imaging-$(PIL_VERSION)/README:
-       wget $(PIL_URL) -O $(PYTHON_DIR)/Extensions/pil.tgz
+       @$(httpget) $(PIL_URL) $(PYTHON_DIR)/Extensions/pil.tgz
        tar -C $(PYTHON_DIR)/Extensions -xzf $(PYTHON_DIR)/Extensions/pil.tgz
        rm -f $(PYTHON_DIR)/Extensions/pil.tgz
 .PHONY: pil-unpack
index afca669..520aa48 100644 (file)
@@ -6,7 +6,7 @@ buildout.cfg:
 ###########################################################################
 
 eggs::
-       $(EASY_INSTALL) ZopeSkel
+       @$(install) ZopeSkel
 
 buildout:: buildout.cfg
 
index af6dc8a..b9db381 100644 (file)
@@ -6,18 +6,19 @@ SETUPTOOLS_URL  ?= http://peak.telecommunity.com/dist/ez_setup.py
 
 ###########################################################################
 
-PYTHON_DIR=python
+PYTHON_DIR ?= python
 
 PYTHON       = $(BASEDIR)/$(PYTHON_DIR)/bin/python
 EASY_INSTALL = $(BASEDIR)/$(PYTHON_DIR)/bin/easy_install
+PIP          = $(BASEDIR)/$(PYTHON_DIR)/bin/pip
 PASTER       = $(BASEDIR)/$(PYTHON_DIR)/bin/paster
 
 ###########################################################################
 
 python-unpack: $(PYTHON_DIR)/Python-$(PYTHON_VERSION)/README
 $(PYTHON_DIR)/Python-$(PYTHON_VERSION)/README:
-       mkdir $(PYTHON_DIR)
-       wget "$(PYTHON_URL)" -O $(PYTHON_DIR)/python.tgz
+       mkdir -p $(PYTHON_DIR)
+       @$(httpget) $(PYTHON_URL) $(PYTHON_DIR)/python.tgz
        tar -C $(PYTHON_DIR) -xzf $(PYTHON_DIR)/python.tgz
        rm -f $(PYTHON_DIR)/python.tgz
 .PHONY: python-unpack
@@ -40,14 +41,30 @@ python-clean:
 python: python-unpack python-build
 .PHONY: python
 
-setuptools: $(EASY_INSTALL)
-$(EASY_INSTALL):
-       mkdir $(PYTHON_DIR)/Extensions
-       wget $(SETUPTOOLS_URL) -O $(PYTHON_DIR)/Extensions/ez_setup.py
+setuptools: $(BASEDIR)/$(PYTHON_DIR)/bin/easy_install
+$(BASEDIR)/$(PYTHON_DIR)/bin/easy_install:
+       mkdir -p $(PYTHON_DIR)/Extensions
+       @$(httpget) $(SETUPTOOLS_URL) $(PYTHON_DIR)/Extensions/ez_setup.py
+       @(                                                                                                              \
+           if [ -r $(DLCACHE)/setuptools-*.egg ]; then                                                                 \
+               name="`cd $(DLCACHE); echo setuptools-*.egg`";                                                          \
+               echo "Fetching '$$name' from cache.";                                                                   \
+               cp "$(DLCACHE)/$$name" $(PYTHON_DIR)/Extensions;                                                        \
+           else                                                                                                        \
+               echo "Downloading setuptools.";                                                                         \
+               ( cd $(PYTHON_DIR)/Extensions && $(PYTHON) -c 'import ez_setup; ez_setup.download_setuptools()' );      \
+               cp $(PYTHON_DIR)/Extensions/setuptools-*.egg $(DLCACHE);                                                \
+           fi;                                                                                                         \
+       )
        cd $(PYTHON_DIR)/Extensions && $(PYTHON) ez_setup.py
 .PHONY: setuptools
 
-pybase: python setuptools
+pip: $(PIP)
+$(PIP):
+       @$(dlegg) pip
+       $(EASY_INSTALL) $(DLCACHE)/pip-*.egg
+
+pybase: python setuptools pip
 .PHONY: pybase
 
 ###########################################################################
@@ -56,7 +73,7 @@ debdepends::
        aptitude install build-essential zlibc
 
 eggs::
-       $(EASY_INSTALL) zc.buildout
+       @$(install) zc.buildout
 
 .gitignore::
        @$(gitignore) /python/
index 5f6edb7..f79d198 100644 (file)
@@ -13,14 +13,20 @@ buildout.cfg:
            echo "zopeproject --no-buildout \"$(PROJECTNAME)\"" &&                                                         \
            (echo "$(ADMINUSER)"; echo "$(ADMINPWD)"; echo "$(EGGDIR)") | $(ZOPEPROJECT) --no-buildout "$(PROJECTNAME)" && \
            mv $(PROJECTNAME)/* $(BASEDIR)/                                                                                \
-       ) || ( rm -f temp-project; false )
-       @(rm -rf $(BASEDIR)/temp-project)
+       ) || ( rm -rf temp-project; false )
        @echo
+       @(rm -rf $(BASEDIR)/temp-project)
+       @sed -i -e '/\[buildout\]/ba' -eb -e:a                  \
+           -ea\\ -e 'download-cache = $(DLCACHE)/downloads'    \
+           -ea\\ -e 'install-from-cache = true'                \
+           -ea\\ -e 'extends-cache = $(DLCACHE)/extends'       \
+                buildout.cfg || ( rm -f buildout.cfg; false )
+       @mkdir -p $(DLCACHE)/downloads $(DLCACHE)/extends || ( rm -f buildout.cfg; false )
 
 ###########################################################################
 
 eggs::
-       $(EASY_INSTALL) zopeproject
+       @$(install) zopeproject
 
 buildout:: buildout.cfg
 
index 70060c2..f718315 100644 (file)
@@ -1,10 +1,10 @@
 # -*- makefile -*-
 
-PROJECTNAME := myproject
+PROJECTNAME := appmain
 
 ###########################################################################
 
-MAKELIB := $(shell for lib in plone-skeleton . ..; do [ ! -r "$$lib"/Makefile.master ] || break; done; echo "`cd $$lib; pwd`" )
+MAKELIB := $(shell for lib in plone-skeleton . ..; do [ ! -r "$$lib"/Makefile.master ] || break; done; echo "`cd $$lib; pwd`")
 
 include $(MAKELIB)/Makefile.master
 include $(MAKELIB)/Makefile.pybase