Commit 85bc102e authored by Noé Gaumont's avatar Noé Gaumont 🐙
Browse files

feat: use python package instead of debian package for cubicweb

related to cubicweb#337
parent ea7e60d8e067
......@@ -13,9 +13,6 @@ RUN apt-get update && apt-get -y --no-install-recommends install \
postgresql-client \
$PYTHON-pip \
$PYTHON-setuptools \
$PYTHON-crypto \
$PYTHON-psycopg2 \
$PYTHON-jinja2 \
pwgen \
&& rm -rf /var/lib/apt/lists/*
RUN test $PYTHON = "python" && export py=python2 pip=pip2 || export py=python3 pip=pip3; \
......@@ -24,9 +21,7 @@ RUN test $PYTHON = "python" && export py=python2 pip=pip2 || export py=python3 p
pip install --upgrade pip
COPY docker-cubicweb-helper check-docker-updates.sh get-cube /usr/local/bin/
RUN chmod +x /usr/local/bin/docker-cubicweb-helper /usr/local/bin/check-docker-updates.sh /usr/local/bin/get-cube
RUN mkdir /repo
RUN echo -n "" | gzip -9c > /repo/Packages.gz
RUN echo "deb [trusted=yes] file:///repo ./" > /etc/apt/sources.list.d/local.list
ENV PIP_NO_CACHE_DIR off
ENV PIP_DISABLE_PIP_VERSION_CHECK on
RUN useradd cubicweb --uid 1000 -m -s /bin/bash
......@@ -49,32 +44,14 @@ ENV CW_DB_DRIVER=postgres
ENV CW_LOGIN=admin
ENV CW_PASSWORD=admin
ENV CW_BASE_URL=http://localhost:8080
ARG CUBICWEB_SOURCE
RUN pip install $CUBICWEB_SOURCE
USER cubicweb
WORKDIR /home/cubicweb
EXPOSE 8080/tcp
ENTRYPOINT ["/entrypoint.sh"]
CMD ["start"]
ARG DIST
FROM logilab/cubicweb:$DIST-buildpackage as build
ARG SOURCE_TARBALL
RUN if test -n "$SOURCE_TARBALL"; then buildpackage -u "$SOURCE_TARBALL"; fi
FROM base
USER root
RUN apt-get update && apt-get -y --no-install-recommends install \
wget \
gnupg \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
ARG DIST
ARG COMPONENT
RUN echo "deb http://apt.logilab.fr $DIST $COMPONENT" > /etc/apt/sources.list.d/logilab.list
RUN wget -O - https://apt.logilab.fr/key.asc | apt-key --keyring /etc/apt/trusted.gpg.d/logilab.gpg add -
COPY --from=build /repo /repo
ARG PACKAGES
RUN apt-get update && apt-get -y --install-recommends install \
$PACKAGES \
&& rm -rf /var/lib/apt/lists/* \
&& apt-mark hold $PACKAGES
USER cubicweb
ARG DIST
FROM debian:$DIST-slim
RUN apt-get update && \
apt-get -y --no-install-recommends install \
wget \
gnupg \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
ARG DIST
RUN echo "deb http://apt.logilab.fr $DIST main" > /etc/apt/sources.list.d/logilab.list
RUN wget -O - https://apt.logilab.fr/key.asc | apt-key --keyring /etc/apt/trusted.gpg.d/logilab.gpg add -
ARG PYTHON
ENV PYTHON=$PYTHON
CMD apt-get update > /dev/null && \
apt-cache show "$PYTHON"-cubicweb | grep '^Version' | cut -f2 -d' '
......@@ -14,30 +14,16 @@ Tags
Images with cubicweb pre-installed:
* ``latest``, ``3.29``, ``py37-buster-3.29``
* ``dev``, ``py37-buster-dev`` (built using latest mercurial changeset)
* ``3.28``, ``py37-buster-3.28``
* ``3.27``, ``py37-buster-3.27``
* ``3.26``, ``py37-buster-3.26``
* ``3.25``, ``py27-buster-3.25``
* ``py35-stretch-3.27``
* ``py35-stretch-3.26``
* ``py27-buster-3.26``
* ``py27-stretch-3.26``
* ``py27-buster-3.25``
* ``py27-stretch-3.25``
Image without cubicweb installed:
* ``py37``, ``py37-buster``
* ``py35``, ``py35-stretch``
* ``py27``, ``py27-buster``
* ``py27-stretch``
* ``latest``, ``3.29``, ``buster-3.29``
* ``dev``, ``buster-dev`` (built using latest mercurial changeset)
* ``3.28``, ``buster-3.28``
* ``3.27``, ``buster-3.27``
* ``3.26``, ``buster-3.26``
Image for building debian packages:
* ``buildpackage``, ``buster-buildpackage``
* ``stretch-buildpackage``
What's included ?
......@@ -114,7 +100,7 @@ onbuild images
.. warning:: A lot of magic happen with onbuild images. They are DEPRECATED.
All images have a ``onbuild`` version by adding the suffix ``-onbuild``.
The single tag ``onbuild`` is an alias for py37-buster-3.29-onbuild.
The single tag ``onbuild`` is an alias for buster-3.29-onbuild.
These images use the `ONBUILD intruction`_ to copy current code to the build
context and install your cube in develop mode and create an instance of your cube.
......@@ -129,10 +115,6 @@ You can even build an image without actually writing any Dockerfile::
echo "FROM logilab/cubicweb:onbuild" | docker build -f - -t cubicweb-blog .
In case you don't want a cubicweb version pre-installed and let your own dependencies control what version to install::
FROM logilab/cubicweb:py37-onbuild
.. _cubicweb-blog: https://hg.logilab.org/master/cubes/blog
......
......@@ -5,8 +5,10 @@ import subprocess
import sys
import logging
from dataclasses import dataclass
from typing import Optional, List, Any
from itertools import product
from typing import Optional, List, Any, Union
import json
from urllib.request import urlopen
from distutils.version import StrictVersion
LOG = logging.getLogger(__name__)
......@@ -16,42 +18,33 @@ CWREPO = "https://forge.extranet.logilab.fr/cubicweb/cubicweb/"
@dataclass
class CubicWebImage:
python: str
debian: str
package_name: str
package_version: str
@property
def cubicweb_version(self):
return self.package_version.split("-", 1)[0]
debian_dist: str
cubicweb_version: Union[StrictVersion, str]
@property
def cubicweb_major_version(self):
return self.cubicweb_version.rsplit(".", 1)[0]
return ".".join(map(str, self.cubicweb_version.version[:2]))
def __le__(self, other):
lversion = [int(v) for v in self.cubicweb_version.split(".")]
rversion = [int(v) for v in other.cubicweb_version.split(".")]
return lversion <= rversion
return self.cubicweb_version <= other.cubicweb_version
def __gt__(self, other):
return not self <= other
def is_dev(self):
return isinstance(self.cubicweb_version, str) and self.cubicweb_version == "dev"
@property
def tag(self):
if self.package_version == "dev":
if self.is_dev():
return f"{REGISTRY}:dev"
return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_version}"
@property
def base_docker_image(self):
return f"{self.python}-{self.debian}"
return f"{REGISTRY}:{self.debian_dist}-{self.cubicweb_version}"
@property
def major_tag(self):
if self.package_version == "dev":
if self.is_dev():
return f"{REGISTRY}:dev"
return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_major_version}"
return f"{REGISTRY}:{self.debian_dist}-{self.cubicweb_major_version}"
def conditionnal_append(
......@@ -111,7 +104,7 @@ def _cwdev(_cache={}):
def get_major_tags(images: List[CubicWebImage]):
latest = {}
for cwimage in images:
if cwimage.package_version == "dev":
if cwimage.is_dev():
continue
version = cwimage.cubicweb_major_version
......@@ -163,17 +156,13 @@ def build_image(image: CubicWebImage, onbuild: Optional[str], no_cache=False) ->
args["FROM"] = image.tag
tag = image.tag + "-onbuild"
else:
args["PYTHON"] = "python" if image.python == "py27" else "python3"
args["PYTHON"] = "python3"
dockerfile = "Dockerfile"
args["DIST"] = image.debian
args["PACKAGES"] = f"{image.package_name}={image.package_version}"
args["DIST"] = image.debian_dist
args["CUBICWEB_SOURCE"] = f"cubicweb[pyramid]=={image.cubicweb_version}"
tag = image.tag
if image.package_version == "dev":
args["SOURCE_TARBALL"] = _cwdev()
args["COMPONENT"] = "main"
args["PACKAGES"] = f"{image.package_name}"
else:
args["COMPONENT"] = "cubicweb-{}".format(image.cubicweb_major_version)
if image.is_dev():
args["CUBICWEB_SOURCE"] = _cwdev()
cmd = [
"docker",
......@@ -217,40 +206,6 @@ def build_buildpackage(dist):
)
def get_cubicweb_images(debian_dists: List[str], python_versions: List[str]):
images = []
for dist, python_version in product(debian_dists, python_versions):
python = "python" if python_version == "py27" else "python3"
check_call(
"docker",
"build",
".",
"-f",
"Dockerfile.getversion",
"-t",
"cubicweb:getversion",
"--build-arg",
f"DIST={dist}",
"--build-arg",
f"PYTHON={python}",
)
versions = check_output(
"docker", "run", "--rm", "cubicweb:getversion"
).splitlines()
# filter release candidate version
valid_versions = [version for version in versions if "~rc" not in version]
for version in valid_versions:
image = CubicWebImage(
python=python_version,
debian=dist,
package_name=f"{python}-cubicweb",
package_version=version,
)
images.append(image)
return images
def build(debian_dists: List[str], images: List[CubicWebImage] = [], rebuild=False):
for dist in debian_dists:
if rebuild:
......@@ -262,7 +217,9 @@ def build(debian_dists: List[str], images: List[CubicWebImage] = [], rebuild=Fal
for image in images:
for image_type in (None, "onbuild"):
built = build_image(image, image_type)
conditionnal_append(built_images, failed_images, built, f"{image}-{image_type}")
conditionnal_append(
built_images, failed_images, built, f"{image}-{image_type}"
)
# Do no try to update image not built
if built and rebuild and image_type is None:
......@@ -304,7 +261,7 @@ def push(images: List[CubicWebImage], last_debian_dist: str):
for major, img in latest.items():
yield f"{REGISTRY}:{major}-onbuild"
yield f"{REGISTRY}:{major}"
if major != 'latest':
if major != "latest":
yield img.major_tag
buildpackage_tags = (
......@@ -318,13 +275,21 @@ def push(images: List[CubicWebImage], last_debian_dist: str):
"push",
tag,
)
conditionnal_append(
succeed_push, fail_push, res.returncode == 0, tag
)
conditionnal_append(succeed_push, fail_push, res.returncode == 0, tag)
print_summary("push image", succeed_push, fail_push)
def get_python_package_versions(package_name: str):
"""
Retrieve all the package version available on pypi
"""
url = f"https://pypi.org/pypi/{package_name}/json"
data = json.load(urlopen(url))
versions = data["releases"].keys()
return versions
if __name__ == "__main__":
import argparse
......@@ -343,14 +308,21 @@ if __name__ == "__main__":
)
args = parser.parse_args()
REGISTRY = args.registry
images = get_cubicweb_images(["stretch", "buster"], ["py27"])
images += get_cubicweb_images(["stretch"], ["py35"])
images += get_cubicweb_images(["buster"], ["py37"])
images.append(CubicWebImage("py37", "buster", "python3-cubicweb", "dev"))
cubicweb_versions = get_python_package_versions("cubicweb")
images = []
for version in cubicweb_versions:
if "rc" in version:
continue
strict_version = StrictVersion(version)
if strict_version < StrictVersion("3.26.0"):
continue
images.append(CubicWebImage("buster", strict_version))
images.append(CubicWebImage("buster", "dev"))
if args.push:
push(images=images, last_debian_dist="buster")
else:
build(
debian_dists=["stretch", "buster"], images=images, rebuild=args.checkrebuild
)
build(debian_dists=["buster"], images=images, rebuild=args.checkrebuild)
tag_aliases(images=images, last_debian_dist="buster")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment