Add {buster,stretch}-buildpackages images

We're going to switch the images to use cubicweb apt repository from apt.logilab.fr

For this introduce a "buildpackage" image which allow to build debian
package(s) and publish them to a local repo in /repo (which can then be copied
from a multi-stage build).

This allow to build docker images without using pip.

See documentation for use-cases.

.dockerignore

@@ -5,3 +5,4 @@
 !check-docker-updates.sh
 !get-cube
 !docker-cubicweb-helper
+!buildpackage

Dockerfile.buildpackage

+ARG DIST
+FROM debian:$DIST-slim
+ARG DIST
+RUN apt-get update && \
+    apt-get -y --no-install-recommends install \
+    bzip2 \
+    wget \
+    gnupg \
+    ca-certificates \
+    devscripts \
+    python3 \
+    python3-setuptools \
+    equivs \
+    && rm -rf /var/lib/apt/lists/*
+RUN echo "deb http://apt.logilab.fr $DIST main" > /etc/apt/sources.list.d/logilab.list
+RUN wget -O - https://apt.logilab.fr/key.asc | apt-key --keyring /etc/apt/trusted.gpg.d/logilab.gpg add -
+RUN mkdir /repo
+RUN echo -n "" | gzip -9c > /repo/Packages.gz
+RUN echo "deb [trusted=yes] file:///repo ./" > /etc/apt/sources.list.d/local.list
+COPY buildpackage /usr/local/bin/
+RUN chmod +x /usr/local/bin/buildpackage

README.rst

@@ -29,6 +29,11 @@ Image without cubicweb installed:
 * ``py27``, ``py27-buster``
 * ``py27-stretch``
 
+Image for building debian packages:
+
+* ``buildpackage``, ``buster-buildpackage``
+* ``stretch-buildpackage``
+
 
 What's included ?
 -----------------
@@ -208,3 +213,33 @@ copying useless files inside the docker image. For example::
 
 Always pull the base image before running the build. Base images are rebuild in
 case of debian security update or new pypi releases.
+
+buildpackage images
+-------------------
+
+These images can be used to build debian package(s) and publish them to a repo
+located in /repo suitable for use within a multi-stage build.
+
+Example, given all dependencies are available on "deb http://apt.logilab.fr buster cubicweb-3.26" repository and given you're working in the source tree of `cubicweb-blog`_::
+
+   FROM logilab/cubicweb:buildpackage as buildpackage
+   COPY . /src
+   RUN buildpackage -d /src
+
+   FROM logilab/cubicweb:3.26
+   COPY --from=buildpackage /repo /repo
+   RUN apt-get update && apt-get -y --no-install-recommends install cubicweb-blog
+   ENV CUBE=blog
+   USER cubicweb
+   RUN docker-cubicweb-helper create-instance
+
+
+If you need to build more packages, or build specific revisions, the
+``buildpackage`` script can also build from an archive::
+
+   FROM logilab/cubicweb:buster-buildpackage
+   RUN buildpackage -u https://hg.logilab.org/master/cubes/comment/archive/tip.tar.gz
+   COPY . /src
+   RUN buildpackage -d /src
+
+   [...]

build.py

@@ -20,6 +20,7 @@ ALIASES = (
     ('3.26', 'py37-buster-3.26'),
     ('dev', 'py37-buster-dev'),
     ('latest', 'py37-buster-3.26'),
+    ('buildpackage', 'buster-buildpackage'),
 )
 
 
@@ -55,6 +56,9 @@ def _tag(*args, registry=None):
 
 def tag_aliases():
     for alias, source in ALIASES:
+        if alias == 'buildpackage':
+            check_call('docker', 'tag', _tag(source), _tag(alias))
+            continue
         for onbuild in [None, 'onbuild']:
             if alias == 'latest' and onbuild == 'onbuild':
                 tag = _tag('onbuild')
@@ -106,11 +110,20 @@ def red(text):
     return '\033[0;31m' + text + '\033[0m'
 
 
+def build_buildpackage(dist):
+    tag = '{}:{}-buildpackage'.format(REGISTRY, dist)
+    check_call('docker', 'build', '-t', tag,
+               '--build-arg', 'DIST={}'.format(dist),
+               '-f', 'Dockerfile.buildpackage', '.')
+
+
 def build(rebuild=False):
     if rebuild:
         # pull base images
         check_call('docker', 'pull', 'debian:stretch-slim')
         check_call('docker', 'pull', 'debian:buster-slim')
+    build_buildpackage('stretch')
+    build_buildpackage('buster')
     for matrix in MATRIX:
         for python, dist, cw, onbuild in itertools.product(*matrix):
             build_image(python, dist, cw, onbuild)

buildpackage

+#!/bin/bash
+
+set -e
+
+usage() {
+    echo "$0 (-d <dir> | -u <url to tar.gz>)"
+}
+
+options=$(getopt hd:u: "$@")
+eval set -- "$options"
+while true;
+do
+    case "$1" in
+        -h) usage && exit 0;;
+        -d) dir=$2; shift 2;;
+        -u) url=$2; shift 2;;
+        *) break;;
+    esac
+done
+(test -z "$dir" && test -z "$url") && usage && exit 64
+(test -n "$dir" && test -n "$url") && usage && exit 64
+
+if test -n "$url"; then
+    src=$(mktemp -d)
+    dst=$(mktemp)$(basename "$url")
+    trap 'rm -rf "$src"' exit
+    wget -q -O "$dst" "$url"
+    tar -C "$src" --strip-component=1 -xf "$dst"
+else
+    src=$dir
+fi
+
+apt-get update
+tmp=$(mktemp -d)
+
+cd "$src" && python3 setup.py sdist
+tar -C "$tmp" --strip-component=1 -xf "$src"/dist/*.tar.gz
+
+pushd $tmp
+cp -r "$src/debian" .
+yes | mk-build-deps -i
+dpkg-buildpackage -us -uc --build=binary
+dcmd cp ../*_amd64.changes /repo/
+cd /repo && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz
+rm -rf /var/lib/apt/lists/*
+rm -rf "$tmp"
+apt-get purge -y \*-build-deps
+apt-get autoremove --purge -y