Commit a35ea483 authored by Philippe Pepiot's avatar Philippe Pepiot
Browse files

Initial release

parents
**
!uwsgi.ini
!pyramid.ini.j2
!entrypoint.sh
!check-docker-updates.sh
!get-cube.sh
ARG DIST
FROM debian:$DIST-slim
ENV LANG C.UTF-8
RUN mkdir -p /usr/share/man/man1 && mkdir -p /usr/share/man/man7
RUN apt-get update && apt-get -y dist-upgrade \
&& rm -rf /var/lib/apt/lists/*
ARG PYTHON
RUN apt-get update && apt-get -y --no-install-recommends install \
gettext \
uwsgi \
uwsgi-plugin-$PYTHON \
graphviz \
postgresql-client \
$PYTHON-pip \
$PYTHON-setuptools \
$PYTHON-crypto \
$PYTHON-psycopg2 \
$PYTHON-jinja2 \
pwgen \
&& rm -rf /var/lib/apt/lists/*
RUN test $PYTHON = "python" && export py=python2 pip=pip2 || export py=python3 pip=pip3; \
update-alternatives --install /usr/bin/python python /usr/bin/$py 50 && \
update-alternatives --install /usr/bin/pip pip /usr/bin/$pip 50
COPY check-docker-updates.sh get-cube.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/check-docker-updates.sh /usr/local/bin/get-cube.sh
ENV PIP_NO_CACHE_DIR off
ENV PIP_DISABLE_PIP_VERSION_CHECK on
# Markdown require setuptools>=36
RUN pip install 'setuptools>=36'
RUN useradd cubicweb --uid 1000 -m -s /bin/bash
RUN install -d -o cubicweb -g cubicweb /etc/cubicweb.d
COPY uwsgi.ini /etc/uwsgi/uwsgi.ini
RUN echo "plugins = http,$PYTHON" >> /etc/uwsgi/uwsgi.ini
COPY pyramid.ini.j2 /
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENV CW_CUBES_PATH=/usr/local/share/cubicweb/cubes
ENV CW_INSTANCE=instance
ENV CW_INSTANCES_DIR=/etc/cubicweb.d
ENV CW_INSTANCES_DATA_DIR=/etc/cubicweb.d
ENV CW_LOG_FILE=/dev/stdout
ENV CW_LOG_THRESHOLD=WARNING
ENV CW_DB_HOST=
ENV CW_DB_USER=cubicweb
ENV CW_DB_PASSWORD=
ENV CW_DB_DRIVER=postgres
ENV CW_LOGIN=admin
ENV CW_PASSWORD=admin
ENV CW_BASE_URL=http://localhost:8080
USER cubicweb
WORKDIR /home/cubicweb
EXPOSE 8080/tcp
ENTRYPOINT ["/entrypoint.sh"]
CMD ["start"]
ARG FROM
FROM $FROM
USER root
ARG CUBICWEB
RUN pip install "$CUBICWEB"
USER cubicweb
ARG FROM
FROM $FROM
ONBUILD USER root
ONBUILD COPY . /src/
ONBUILD RUN pip install -e /src
ONBUILD USER cubicweb
ONBUILD RUN cubicweb-ctl create $(get-cube.sh) $CW_INSTANCE --automatic --no-db-create
pipeline {
agent {label 'debian_stretch'}
stages {
stage('build') {
steps {
sh "python3 build.py --checkrebuild"
}
}
stage('push') {
steps {
sh "python3 build.py --push"
}
}
}
}
CubicWeb docker images
======================
This project helps to build CubicWeb_ based applications as Docker_ images.
Builds include various versions of cubicweb and python and are using Debian_
base images.
.. _CubicWeb: https://www.cubicweb.org/
.. _Docker: https://www.docker.com/
.. _Debian: https://www.debian.org/
Tags
----
Images with cubicweb pre-installed:
* ``latest``, ``3.26``, ``py37-buster-3.26``
* ``onbuild``, ``3.26-onbuild``, ``py37-buster-3.26-onbuild``
* ``dev``, ``py37-buster-dev`` (built using latest mercurial changeset)
* ``3.25``, ``py27-buster-3.25``
* ``py35-stretch-dev``
* ``py35-stretch-3.26``
* ``py27-buster-3.26``
* ``py27-stretch-3.25``
* ``py27-stretch-3.26``
Image without cubicweb installed:
* ``py37``, ``py37-buster``
* ``py35``, ``py35-stretch``
* ``py27``, ``py27-buster``
* ``py27-stretch``
What's included ?
-----------------
The image include required packages to build a cubicweb application image
suitable for production. It's designed to be used as a parent image for your
application Dockerfile.
The image contains:
* python with psycopg2
* /usr/bin/python and /usr/bin/pip are symlink to the selected python version (2.7, 3.5 or 3.7)
* gettex, graphviz
* uwsgi with a config file in /etc/uwsgi/uwsgi.ini
* a pyramid configuration templated from environment variables in /pyramid.ini
* a entrypoint to configure pyramid.ini, static data files in
/etc/cubicweb.d/instance/data and automatic upgrade of database.
The entrypoint can also be used to run various command like database creation
with ``db-create`` or to run arbitrary commands.
The image has some expectations:
* The top level cube source code is expected to be found in /src directory. In
case you install your cube elsewhere of from a pypi release, you will have to
set the ``CUBE`` environment variable.
* The default instance name is "instance" and its configuration directory is in
/etc/cubicweb.d/instance, this can be changed with the ``CW_INSTANCE``
environment variable. Although you shouldn't need to modify this.
How to build an image for a cubicweb application ?
--------------------------------------------------
There's multiple ways of using theses images corresponding to differents level
of integration.
Here's some hints to make the best choice:
* Images with cubicweb pre-installed build faster
* Images without cubicweb pre-installed allow to use your own version of cubicweb
* ``onbuild`` images are useful when the Dockerfile in versioned in the source
tree, except when your cube require a build toolchain to install.
onbuild images
~~~~~~~~~~~~~~
All images have a ``onbuild`` version by adding the suffix ``-onbuild``.
The single tag ``onbuild`` is an alias for py37-buster-3.26-onbuild.
These images use the `ONBUILD intruction`_ to copy current code to the build
context and install your cube in develop mode and create an instance of your cube.
.. _ONBUILD intruction: https://docs.docker.com/engine/reference/builder/#onbuild
For example, given you're in the source tree of `cubicweb-blog`_, a Dockerfile would be as simple as::
FROM hub.extranet.logilab.fr/cubicweb/cubicweb:onbuild
You can even build an image without actually writing any Dockerfile::
echo "FROM hub.extranet.logilab.fr/cubicweb/cubicweb:onbuild" | docker build -f - -t cubicweb-blog .
In case you don't want a cubicweb version pre-installed and let your own dependencies control what version to install::
FROM hub.extranet.logilab.fr/cubicweb/cubicweb:py37-onbuild
.. _cubicweb-blog: https://hg.logilab.org/master/cubes/blog
other images
~~~~~~~~~~~~
This is useful for bulding images out of the source tree, or when you require
additional dependencies to bootstrap the instance.
In this case you have to:
* install your cube and dependencies
* bootstrap an instance in /etc/cubicweb.d/instance
For example, given you're in the source tree of `cubicweb-blog`_::
FROM hub.extranet.logilab.fr/cubicweb/cubicweb:3.26
USER root
COPY . /src
RUN pip install -e /src
USER cubicweb
RUN cubicweb-ctl create blog $CW_INSTANCE --automatic --no-db-create
In case of out-of-source tree or not installing from /src directory, you will
also have to set the ``CUBE`` environment variable::
FROM hub.extranet.logilab.fr/cubicweb/cubicweb:3.26
USER root
RUN pip install cubicweb-blog
USER cubicweb
RUN cubicweb-ctl create blog $CW_INSTANCE --automatic --no-db-create
ENV CUBE=blog
How to run resulting images ?
-----------------------------
Environment variables control settings from ``all-in-one.conf``, most important are:
* ``CW_BASE_URL`` (default http://localhost:8080) should be set to the final url the instance will be accessed
with. Including the scheme, for example: ``https://myapp.demo.logilab.org``
* ``CW_DB_DRIVER`` (default postgres), ``CW_DB_NAME``, ``CW_DB_USER`` and
``CW_DB_PASSWORD`` control database settings
* ``CW_LOGIN`` and ``CW_PASSWORD`` control the admin login and password,
default to admin:admin
Quick testing using non persistent sqlite database
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This can be used to validate your image actually works::
docker run --rm -it -e CW_DB_NAME=db.sqlite myimage sh -c "cubicweb-ctl db-create -a instance && uwsgi --ini /etc/uwsgi/uwsgi.ini"
Then go to http://localhost:8080 to access your instance.
Create initial database
~~~~~~~~~~~~~~~~~~~~~~~
To create the initial database you will have to run ``db-create`` first:
For example::
# Create the database on local postgresql cluster in a database named "myapp"
docker run --rm -it -e CW_DB_NAME=myapp CW_DB_USER=me -v /var/run/postgresql:/var/run/postgresql myimage db-create
# Create the database on remote postgresql server in a database named "myapp"
docker run --rm -it -e CW_DB_NAME=myapp CW_DB_USER=me CW_DB_PASSWORD=secret CW_DB_HOST=dbserver myimage db-create
# Create the database in a local /tmp/db.sqlite file
docker run --rm -it -e CW_DB_NAME=/tmp/db.sqlite -v /tmp/db.sqlite:/tmp/db.sqlite myimage db-create
Run wsgi server
~~~~~~~~~~~~~~~
To start uwsgi server on local port 8080::
# run foreground
docker run --rm -it -p 8080:8080 myapp
# run in background
docker run -d --restart=always --name myapp myapp
Run looping tasks (aka scheduler)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
To run cubicweb looping tasks, you will also have to start the ``scheduler``::
# run foreground
docker run --rm -it myapp cubicweb-ctl scheduler instance
# run in background
docker run -d --restart=always --name myapp-scheduler myapp cubicweb-ctl scheduler instance
#!/usr/bin/env python3
import itertools
import subprocess
import sys
import logging
LOG = logging.getLogger(__name__)
REGISTRY = 'hub.extranet.logilab.fr/cubicweb/cubicweb'
CWREPO = 'https://hg.logilab.org/master/cubicweb'
MATRIX = [
(['py27'], ['stretch', 'buster'], [None, '3.25', '3.26'],
[None, 'onbuild']),
(['py35'], ['stretch'], [None, '3.26', 'dev'], [None, 'onbuild']),
(['py37'], ['buster'], [None, '3.26', 'dev'], [None, 'onbuild']),
]
ALIASES = (
('3.25', 'py27-buster-3.25'),
('3.26', 'py37-buster-3.26'),
('dev', 'py37-buster-dev'),
('latest', 'py37-buster-3.26'),
)
def run(*args):
LOG.info('%s', ' '.join(args))
return subprocess.run(args, stdout=subprocess.PIPE)
def check_call(*args):
LOG.info('%s', ' '.join(args))
return subprocess.check_call(args)
def check_output(*args):
LOG.info('%s', ' '.join(args))
return subprocess.check_output(args).decode().strip()
def _cwdev(_cache={}):
if not _cache:
rev = check_output('hg', 'id', '-r', 'default', CWREPO)
_cache[None] = '{}/archive/{}.tar.gz#egg=cubicweb[pyramid]'.format(
CWREPO, rev)
return _cache[None]
def _tag(*args, registry=None):
tag_part = '-'.join([x for x in args if x is not None])
if tag_part:
return '{}:{}'.format(registry or REGISTRY, tag_part)
return REGISTRY
def tag_aliases():
for alias, source in ALIASES:
for onbuild in [None, 'onbuild']:
if alias == 'latest' and onbuild == 'onbuild':
tag = _tag('onbuild')
else:
tag = _tag(alias, onbuild)
src = _tag(source, onbuild)
check_call('docker', 'tag', src, tag)
def get_cubicweb_arg(cw):
return {
'3.25': 'cubicweb[pyramid]>=3.25,<3.26',
'3.26': 'cubicweb[pyramid]>=3.26,<3.27',
'dev': _cwdev(),
}[cw]
def build_image(python, dist, cw, onbuild, no_cache=False, registry=None):
tag = _tag(python, dist, cw, onbuild, registry=registry)
args = {}
dockerfile = 'Dockerfile'
if onbuild is not None:
dockerfile = 'Dockerfile.onbuild'
args['FROM'] = _tag(python, dist, cw)
elif cw is not None:
dockerfile = 'Dockerfile.cubicweb'
args['FROM'] = _tag(python, dist)
args['CUBICWEB'] = get_cubicweb_arg(cw)
else:
args['DIST'] = dist
args['PYTHON'] = 'python' if python == 'py27' else 'python3'
cmd = [
'docker', 'build', '-t', tag,
'-f', dockerfile,
]
if no_cache:
cmd += ['--no-cache']
for key, value in args.items():
cmd += ['--build-arg', '{}={}'.format(key, value)]
cmd += ['.']
check_call(*cmd)
def green(text):
return '\033[0;32m' + text + '\033[0m'
def red(text):
return '\033[0;31m' + text + '\033[0m'
def build(rebuild=False):
for matrix in MATRIX:
for python, dist, cw, onbuild in itertools.product(*matrix):
build_image(python, dist, cw, onbuild)
if rebuild and cw is None and onbuild is None:
tag = _tag(python, dist)
out = run(
'docker', 'run', '--rm', '-t',
'--user', 'root',
'--entrypoint', 'check-docker-updates.sh',
tag, 'apt')
if out.returncode == 1:
LOG.warning(red(
'%s debian packages updates are available: %s'),
tag, out.stdout)
build_image(python, dist, cw, onbuild, no_cache=True)
else:
assert (out.returncode, out.stdout) == (0, b''), out
LOG.info(green('%s debian packages are up-to-date'), tag)
if rebuild and cw is not None and onbuild is None:
tag = _tag(python, dist, cw)
out = run(
'docker', 'run', '--rm', '-t',
'--user', 'root',
'--entrypoint', 'check-docker-updates.sh',
tag, 'pip', get_cubicweb_arg(cw))
if out.returncode == 1:
LOG.info(
red('%s python packages updates are available: %s'),
tag, out.stdout)
build_image(python, dist, cw, onbuild, no_cache=True)
else:
assert (out.returncode, out.stdout) == (0, b''), out
LOG.info(green('%s python packages are up-to-date'), tag)
def push():
for matrix in MATRIX:
for python, dist, cw, onbuild in itertools.product(*matrix):
tag = _tag(python, dist, cw, onbuild)
check_call('docker', 'push', tag)
for alias, source in ALIASES:
for onbuild in [None, 'onbuild']:
if alias == 'latest' and onbuild == 'onbuild':
tag = _tag('onbuild')
else:
tag = _tag(alias, onbuild)
check_call('docker', 'push', tag)
if __name__ == '__main__':
import argparse
logging.basicConfig(format='%(asctime)-15s %(message)s')
LOG.setLevel(logging.INFO)
parser = argparse.ArgumentParser(sys.argv[0])
parser.add_argument('--registry', action='store', default=REGISTRY)
parser.add_argument(
'--checkrebuild', action='store_true', default=False,
help='Rebuild images in case of outdated debian or python package')
parser.add_argument(
'--push', action='store_true', default=False,
help='push images')
args = parser.parse_args()
REGISTRY = args.registry
if args.push:
push()
else:
build(args.checkrebuild)
tag_aliases()
#!/bin/sh
set -e
case "$1" in
apt)
apt-get update >/dev/null
apt list --upgradable 2>/dev/null | grep "upgradable from" && exit 1 || exit 0
;;
pip)
pip freeze > /tmp/freeze.txt
pip install --force-reinstall "$2" >/dev/null
pip freeze > /tmp/freeze.new.txt
diff -u /tmp/freeze.txt /tmp/freeze.new.txt
exit $?
;;
*)
echo "Usage: $0 apt|pip"
exit 64
;;
esac
#!/bin/sh
set -e
export CUBE=$(get-cube.sh)
test -z "$CUBE" && exit 1
if test -z "$CW_DB_NAME"; then
export CW_DB_NAME=$CUBE
fi
PINI=$CW_INSTANCES_DIR/$CW_INSTANCE/pyramid.ini
if ! test -e $PINI; then
test -z "$PYRAMID_SESSION_SECRET" && export PYRAMID_SESSION_SECRET=$(pwgen -s 20)
test -z "$PYRAMID_AUTHTKT_SESSION_SECRET" && export PYRAMID_AUTHTKT_SESSION_SECRET=$(pwgen -s 20)
test -z "$PYRAMID_AUTHTKT_PERSISTENT_SECRET" && export PYRAMID_AUTHTKT_PERSISTENT_SECRET=$(pwgen -s 20)
python > $PINI << EOF
import os
import jinja2
print(jinja2.Template(open('/pyramid.ini.j2').read()).render(os.environ))
EOF
fi
check_upgrade() {
echo "check the instance is properly bootstraped and up to date"
echo "print('OK')" > /tmp/chk.py
isok=$(cubicweb-ctl shell $CW_INSTANCE /tmp/chk.py || true)
if [ "x$isok" != xOK ]; then
echo $isok
echo "upgrading instance"
cubicweb-ctl upgrade --nostartstop --backup-db=no --force --verbosity=0 $CW_INSTANCE
fi
cubicweb-ctl gen-static-datadir $CW_INSTANCE
cubicweb-ctl i18ninstance $CW_INSTANCE
}
case "$1" in
start)
check_upgrade
exec uwsgi --ini /etc/uwsgi/uwsgi.ini
;;
db-create)
exec cubicweb-ctl db-create --automatic $CW_INSTANCE
;;
-h|--help|help)
cat << EOF
Docker image for cubicweb-$CUBE
Usage:
start upgrade and start instance
db-create initialize database
Environment variables:
CW_BASE_URL site url (default http://localhost:8080)
CW_LOG_THRESHOLD log level (default WARNING)
CW_DB_HOST database host
CW_DB_USER database user (default cubicweb)
CW_DB_PASSWORD database password
CW_DB_NAME database name (default $CUBE)
CW_DB_DRIVER database driver (default postgres)
PYRAMID_SESSION_SECRET
PYRAMID_AUTHTKT_SESSION_SECRET
PYRAMID_AUTHTKT_PERSISTENT_SECRET
EOF
exit 64
;;
*)
exec "$@"
;;
esac
#!/bin/sh
set -e
if test -n "$CUBE"; then
echo "$CUBE"
elif test -f /src/setup.py; then
python /src/setup.py --name | sed 's@cubicweb-@@g'
else
>&2 echo "unable to deternine the cube name. You have to set it by the CUBE environment variable"
exit 1
fi
{% set secure = "yes" if CW_BASE_URL.startswith('https://') else "no" -%}
[main]
cubicweb.includes =
cubicweb.pyramid.auth
cubicweb.pyramid.login
cubicweb.session.secret = {{ PYRAMID_SESSION_SECRET }}
cubicweb.auth.authtkt.session.secret = {{ PYRAMID_AUTHTKT_SESSION_SECRET }}
cubicweb.auth.authtkt.session.secure = {{ secure }}
cubicweb.auth.authtkt.persistent.secret = {{ PYRAMID_AUTHTKT_PERSISTENT_SECRET }}
cubicweb.auth.authtkt.persistent.secure = {{ secure }}
[uwsgi]
master = true
http = 0.0.0.0:8080
module = cubicweb.pyramid:wsgi_application()
processes = 2
threads = 8
auto-procname = true
lazy-apps = true
log-master = true
disable-logging = true
http-timeout = 180
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment