Commit 23804138 authored by Noé Gaumont's avatar Noé Gaumont 🐙
Browse files

feat: create image cubicweb-base

--
BREAKING CHANGE
parent 2c308cdb99f8
Pipeline #79549 passed with stage
in 1 minute and 38 seconds
---
stages:
- setup
- triggers
generate-config:
stage: setup
image: python
before_script:
- pip install jinja2
- build
cubicweb-base-test-build:
stage: build
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
rules:
- if: '$CI_MERGE_REQUEST_ID'
script:
- python3 ./generate_gitlab_ci.py
artifacts:
paths:
- gitlab-ci-generated.yml
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
- /kaniko/executor --context $CI_PROJECT_DIR
--dockerfile $CI_PROJECT_DIR/Dockerfile
--build-arg DIST=buster
--no-push
trigger-build:
stage: triggers
trigger:
include:
- artifact: gitlab-ci-generated.yml
job: generate-config
cubicweb-base-build-push:
stage: build
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
rules:
- if: '$CI_MERGE_REQUEST_ID'
when: never
- when: on_success
script:
- |
echo "{\"auths\": {
\"$HUB_REGISTRY\":{\"username\":\"$HUB_REGISTRY_USER\",\"password\":\"$HUB_REGISTRY_PASSWORD\"},
\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}
}}" > /kaniko/.docker/config.json
- /kaniko/executor --context $CI_PROJECT_DIR
--dockerfile $CI_PROJECT_DIR/Dockerfile
--build-arg DIST=buster
--destination $HUB_REGISTRY_IMAGE:buster-1.0
--destination $HUB_REGISTRY_IMAGE:latest
--destination $CI_REGISTRY_IMAGE:buster-1.0
--destination $CI_REGISTRY_IMAGE:latest
......@@ -18,8 +18,8 @@ RUN export py=python3 pip=pip3; \
update-alternatives --install /usr/bin/python python /usr/bin/$py 50 && \
update-alternatives --install /usr/bin/pip pip /usr/bin/$pip 50 && \
pip install --upgrade pip
COPY docker-cubicweb-helper check-docker-updates.sh get-cube /usr/local/bin/
RUN chmod +x /usr/local/bin/docker-cubicweb-helper /usr/local/bin/check-docker-updates.sh /usr/local/bin/get-cube
COPY docker-cubicweb-helper /usr/local/bin/
RUN chmod +x /usr/local/bin/docker-cubicweb-helper
ENV PIP_NO_CACHE_DIR off
ENV PIP_DISABLE_PIP_VERSION_CHECK on
......@@ -46,7 +46,7 @@ ENV CW_BASE_URL=http://localhost:8080
RUN pip install --no-cache-dir psycopg2-binary jinja2
ARG CUBICWEB_SOURCE
RUN pip install --no-cache-dir $CUBICWEB_SOURCE
RUN if [ ! -z "$CUBICWEB_SOURCE" ] ; then pip install --no-cache-dir $CUBICWEB_SOURCE ; fi
USER cubicweb
WORKDIR /home/cubicweb
......
......@@ -12,19 +12,12 @@ base images.
Tags
----
Images with cubicweb pre-installed:
Images available:
* ``latest``, ``3.29``, ``buster-3.29``
* ``dev``, ``buster-dev`` (built using latest mercurial changeset)
* ``3.28``, ``buster-3.28``
* ``3.27``, ``buster-3.27``
* ``3.26``, ``buster-3.26``
* ``latest``,
* ``1.0``,
Image for building debian packages:
* ``buildpackage``, ``buster-buildpackage``
What's included ?
-----------------
......@@ -73,7 +66,7 @@ Example
For example, given you're in the source tree of `cubicweb-blog`_::
FROM logilab/cubicweb:3.29
FROM logilab/cubicweb-base
USER root
COPY . /src
RUN pip install -e /src
......@@ -84,7 +77,7 @@ For example, given you're in the source tree of `cubicweb-blog`_::
In case of out-of-source tree or not installing from /src directory, you will
also have to set the ``CUBE`` environment variable::
FROM logilab/cubicweb:3.29
FROM logilab/cubicweb-base
USER root
RUN pip install cubicweb-blog
USER cubicweb
......
#!/bin/sh
set -e
apt-get update >/dev/null
# Return true if at least one package EXCEPT cubicweb needs to be upgraded
# cubicweb package is mark as hold in order to have container for each version (minor and patch)
apt list --upgradable 2>/dev/null | grep "upgradable from" | grep --invert-match "cubicweb" && exit 1 || exit 0
#!/usr/bin/env python3
import subprocess
from dataclasses import dataclass
from typing import Optional, List, Union
import json
from urllib.request import urlopen
from distutils.version import StrictVersion
from jinja2 import Environment
jinja2_env = Environment()
CWREPO = "https://forge.extranet.logilab.fr/cubicweb/cubicweb/"
LATEST_DEBIAN_DIST = "buster"
JOB_TEMPLATE = """
{{ job_name }}:
stage: build
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
script:
{% raw %}
- echo "{\\"auths\\":{\\"$HUB_REGISTRY\\":{\\"username\\":\\"$HUB_REGISTRY_USER\\",\\"password\\":\\"$HUB_REGISTRY_PASSWORD\\"}}}" > /kaniko/.docker/config.json
{% endraw %}
- /kaniko/executor --context $CI_PROJECT_DIR
--dockerfile $CI_PROJECT_DIR/Dockerfile
--build-arg DIST={{ DIST }}
--build-arg CUBICWEB_SOURCE={{ CUBICWEB_SOURCE }}
{%- for target_tag in TARGETS %}
--destination $HUB_REGISTRY_IMAGE:{{ target_tag }}
{%- endfor %}
"""
@dataclass
class CubicWebImage:
debian_dist: str
cubicweb_version: Union[StrictVersion, str]
@property
def cubicweb_major_version(self):
return ".".join(map(str, self.cubicweb_version.version[:2]))
def __le__(self, other):
return self.cubicweb_version <= other.cubicweb_version
def __gt__(self, other):
return not self <= other
def __hash__(self):
return hash(self.tag)
def is_dev(self):
return isinstance(self.cubicweb_version, str) and self.cubicweb_version == "dev"
@property
def tag(self):
if self.is_dev():
return "dev"
return f"{self.debian_dist}-{self.cubicweb_version}"
@property
def minor_tag(self):
if self.is_dev():
return "dev"
return f"{self.cubicweb_version}"
@property
def major_tag(self):
if self.is_dev():
return "dev"
return f"{self.debian_dist}-{self.cubicweb_major_version}"
def _cwdev(_cache={}):
if not _cache:
rev = (
subprocess.check_output(("hg", "id", "-r", "default", CWREPO))
.decode()
.strip()
)
# url anchor is used to invalidate docker cache.
_cache[
None
] = "{0}-/archive/branch/default/cubicweb-branch-default.tar.gz#rev={1}".format(
CWREPO, rev
)
return _cache[None]
def get_major_tags_by_image(images: List[CubicWebImage]):
latest = {}
for cwimage in images:
if cwimage.is_dev():
continue
if cwimage.debian_dist != LATEST_DEBIAN_DIST:
continue
version = cwimage.cubicweb_major_version
if version not in latest:
latest[version] = cwimage
elif latest[version] <= cwimage:
latest[version] = cwimage
last_major = max(latest.values())
latest["latest"] = last_major
# reverse latest dictionary, to have a image to tags dictionary
image_to_extra_tags = {}
for tag, image in latest.items():
image_to_extra_tags.setdefault(image, []).append(tag)
return image_to_extra_tags
def generate_job_definition_for_image(
image: CubicWebImage,
extra_tags: Optional[List[str]] = None,
) -> None:
args = {}
args["DIST"] = image.debian_dist
args["CUBICWEB_SOURCE"] = f"cubicweb[pyramid]=={image.cubicweb_version}"
tags = [image.tag]
if extra_tags is not None:
tags += extra_tags
if image.is_dev():
args["CUBICWEB_SOURCE"] = _cwdev()
job_definition = jinja2_env.from_string(JOB_TEMPLATE).render(
job_name=f"{image.debian_dist} {image.cubicweb_version}",
TARGETS=tags,
**args,
)
return job_definition
def generate_gitlab_ci(images: List[CubicWebImage] = []):
major_tags = get_major_tags_by_image(images)
with open("gitlab-ci-generated.yml", "w") as gitlab_ci:
for image in images:
job_definition = generate_job_definition_for_image(
image, extra_tags=major_tags.get(image, [])
)
gitlab_ci.write(job_definition)
def get_python_package_versions(package_name: str):
"""
Retrieve all the package version available on pypi
"""
url = f"https://pypi.org/pypi/{package_name}/json"
data = json.load(urlopen(url))
versions = data["releases"].keys()
return versions
if __name__ == "__main__":
cubicweb_versions = get_python_package_versions("cubicweb")
images = []
for version in cubicweb_versions:
if "rc" in version:
continue
strict_version = StrictVersion(version)
if strict_version < StrictVersion("3.26.0"):
continue
images.append(CubicWebImage("buster", strict_version))
images.append(CubicWebImage("buster", "dev"))
generate_gitlab_ci(images)
#!/bin/sh
>&2 echo "get-cube is DEPRECATED, use 'docker-cubicweb-helper get-cube' instead"
exec docker-cubicweb-helper get-cube
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment