Commit a8b0b954 authored by Arthur Lutz's avatar Arthur Lutz
Browse files

feat(.gitlab-ci): Construct the appropriate gitlab-ci.yml instead of running...

feat(.gitlab-ci): Construct the appropriate gitlab-ci.yml instead of running all docker commands locally

The CI will handle the build and push processes

BREAKING CHANGE
* Do not build Docker images (only update .gitlab-ci file)
* Remove every functions from error logging and subproces because handled by the CI
parent 747b5d4b1857
stages:
- setup
- triggers
generate-config:
stage: setup
image: python
before_script:
- pip install jinja2
script:
- python3 ./generate_gitlab_ci.py
artifacts:
paths:
- gitlab-ci-generated.yml
trigger-build:
stage: triggers
trigger:
include:
- artifact: gitlab-ci-generated.yml
job: generate-config
#!/usr/bin/env python3
import itertools
import subprocess
import sys
import logging
from dataclasses import dataclass
from typing import Optional, List, Any, Union
from typing import Optional, List, Union
import json
from urllib.request import urlopen
from distutils.version import StrictVersion
from jinja2 import Environment
jinja2_env = Environment()
LOG = logging.getLogger(__name__)
REGISTRY = "logilab/cubicweb"
CWREPO = "https://forge.extranet.logilab.fr/cubicweb/cubicweb/"
LATEST_DEBIAN_DIST = "buster"
JOB_TEMPLATE = """
{{ job_name }}:
stage: build
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
script:
{% raw %}
- echo "{\\"auths\\":{\\"$HUB_REGISTRY\\":{\\"username\\":\\"$HUB_REGISTRY_USER\\",\\"password\\":\\"$HUB_REGISTRY_PASSWORD\\"}}}" > /kaniko/.docker/config.json
{% endraw %}
- /kaniko/executor --context $CI_PROJECT_DIR
--dockerfile $CI_PROJECT_DIR/Dockerfile
--build-arg DIST={{ DIST }}
--build-arg CUBICWEB_SOURCE={{ CUBICWEB_SOURCE }}
{%- for target_tag in TARGETS %}
--destination $HUB_REGISTRY_IMAGE:{{ target_tag }}
{%- endfor %}
"""
@dataclass
......@@ -31,73 +49,38 @@ class CubicWebImage:
def __gt__(self, other):
return not self <= other
def __hash__(self):
return hash(self.tag)
def is_dev(self):
return isinstance(self.cubicweb_version, str) and self.cubicweb_version == "dev"
@property
def tag(self):
if self.is_dev():
return f"{REGISTRY}:dev"
return f"{REGISTRY}:{self.debian_dist}-{self.cubicweb_version}"
return "dev"
return f"{self.debian_dist}-{self.cubicweb_version}"
@property
def minor_tag(self):
if self.is_dev():
return f"{REGISTRY}:dev"
return f"{REGISTRY}:{self.cubicweb_version}"
return "dev"
return f"{self.cubicweb_version}"
@property
def major_tag(self):
if self.is_dev():
return f"{REGISTRY}:dev"
return f"{REGISTRY}:{self.debian_dist}-{self.cubicweb_major_version}"
def conditionnal_append(
true_tab: List[Any], false_tab: List[Any], condition: bool, value: Any
):
if condition:
true_tab.append(value)
else:
false_tab.append(value)
def print_summary(action_type: str, succeed: List[Any], failed: List[Any]):
LOG.info(f"########### {action_type.upper()} SUMMARY #############")
LOG.info(f"Succeed {action_type.lower()}:")
for elem in succeed:
LOG.info(f" -✅ {elem}")
if failed:
LOG.warning(f"Failed {action_type.lower()}:")
for elem in failed:
LOG.warning(red(f" -❌ {elem}"))
else:
LOG.info(f"All {action_type.lower()} succeed. Good Job !")
def run(*args):
LOG.info("%s", " ".join(args))
# Capture both stdout and stder
result = subprocess.run(args, capture_output=True)
if result.returncode != 0:
LOG.warning(result.stdout.decode())
LOG.warning(red(result.stderr.decode()))
return result
def check_call(*args):
LOG.info("%s", " ".join(args))
return subprocess.check_call(args)
def check_output(*args):
LOG.info("%s", " ".join(args))
return subprocess.check_output(args).decode().strip()
return "dev"
return f"{self.debian_dist}-{self.cubicweb_major_version}"
def _cwdev(_cache={}):
if not _cache:
rev = check_output("hg", "id", "-r", "default", CWREPO)
rev = (
subprocess.check_output(("hg", "id", "-r", "default", CWREPO))
.decode()
.strip()
)
# url anchor is used to invalidate docker cache.
_cache[
None
......@@ -107,12 +90,15 @@ def _cwdev(_cache={}):
return _cache[None]
def get_major_tags(images: List[CubicWebImage]):
def get_major_tags_by_image(images: List[CubicWebImage]):
latest = {}
for cwimage in images:
if cwimage.is_dev():
continue
if cwimage.debian_dist != LATEST_DEBIAN_DIST:
continue
version = cwimage.cubicweb_major_version
if version not in latest:
latest[version] = cwimage
......@@ -120,143 +106,44 @@ def get_major_tags(images: List[CubicWebImage]):
latest[version] = cwimage
last_major = max(latest.values())
latest["latest"] = last_major
return latest
def tag_aliases(images: List[CubicWebImage], last_debian_dist: str):
latest = get_major_tags(images)
tags = []
fail_tags = []
for major, img in latest.items():
tag = f"{REGISTRY}:{major}"
src = img.tag
res = run("docker", "tag", src, tag)
conditionnal_append(tags, fail_tags, res.returncode == 0, tag)
res = run("docker", "tag", src, img.major_tag)
conditionnal_append(tags, fail_tags, res.returncode == 0, img.major_tag)
# tag {last_debian_dist}-3.26.22 as 3.26.22
for img in images:
if img.debian_dist == last_debian_dist:
res = run("docker", "tag", img.tag, img.minor_tag)
conditionnal_append(tags, fail_tags, res.returncode == 0, img.minor_tag)
res = run(
"docker",
"tag",
f"{REGISTRY}:{last_debian_dist}-buildpackage",
f"{REGISTRY}:buildpackage",
)
conditionnal_append(
tags, fail_tags, res.returncode == 0, f"{REGISTRY}:buildpackage"
)
print_summary("tag", tags, fail_tags)
# reverse latest dictionary, to have a image to tags dictionary
image_to_extra_tags = {}
for tag, image in latest.items():
image_to_extra_tags.setdefault(image, []).append(tag)
return image_to_extra_tags
def build_image(image: CubicWebImage, no_cache=False) -> bool:
def generate_job_definition_for_image(
image: CubicWebImage,
extra_tags: Optional[List[str]] = None,
) -> None:
args = {}
dockerfile = "Dockerfile"
args["DIST"] = image.debian_dist
args["CUBICWEB_SOURCE"] = f"cubicweb[pyramid]=={image.cubicweb_version}"
tag = image.tag
tags = [image.tag]
if extra_tags is not None:
tags += extra_tags
if image.is_dev():
args["CUBICWEB_SOURCE"] = _cwdev()
cmd = [
"docker",
"build",
"-t",
tag,
"-f",
dockerfile,
]
if no_cache:
cmd += ["--no-cache"]
for key, value in args.items():
cmd += ["--build-arg", "{}={}".format(key, value)]
cmd += ["."]
result = run(*cmd)
if result.returncode != 0:
return False
return True
def green(text):
return "\033[0;32m" + text + "\033[0m"
def red(text):
return "\033[0;31m" + text + "\033[0m"
def build(debian_dists: List[str], images: List[CubicWebImage] = [], rebuild=False):
built_images = []
failed_images = []
for image in images:
built = build_image(image)
conditionnal_append(built_images, failed_images, built, image)
# Do no try to update image not built
if built and rebuild:
tag = image.tag
out = run(
"docker",
"run",
"--rm",
"-t",
"--user",
"root",
"--entrypoint",
"check-docker-updates.sh",
tag,
"apt",
)
if out.returncode == 1:
LOG.warning(
red("%s debian packages updates are available: %s"),
tag,
out.stdout,
)
build_image(image, no_cache=True)
else:
assert (out.returncode, out.stdout) == (0, b""), out
LOG.info(green("%s debian packages are up-to-date"), tag)
print_summary("build image", built_images, failed_images)
def push(images: List[CubicWebImage], last_debian_dist: str):
latest = get_major_tags(images)
succeed_push = []
fail_push = []
minor_tags = (image.minor_tag for image in images)
minor_tags_with_debian = (image.tag for image in images)
def major_tags_gen():
for major, img in latest.items():
yield f"{REGISTRY}:{major}"
if major != "latest":
yield img.major_tag
buildpackage_tags = (
f"{REGISTRY}:{last_debian_dist}-buildpackage",
f"{REGISTRY}:buildpackage",
job_definition = jinja2_env.from_string(JOB_TEMPLATE).render(
job_name=f"{image.debian_dist} {image.cubicweb_version}",
TARGETS=tags,
**args,
)
return job_definition
for tag in itertools.chain(
minor_tags, minor_tags_with_debian, major_tags_gen(), buildpackage_tags
):
res = run(
"docker",
"push",
tag,
)
conditionnal_append(succeed_push, fail_push, res.returncode == 0, tag)
print_summary("push image", succeed_push, fail_push)
def generate_gitlab_ci(images: List[CubicWebImage] = []):
major_tags = get_major_tags_by_image(images)
with open("gitlab-ci-generated.yml", "w") as gitlab_ci:
for image in images:
job_definition = generate_job_definition_for_image(
image, extra_tags=major_tags.get(image, [])
)
gitlab_ci.write(job_definition)
def get_python_package_versions(package_name: str):
......@@ -270,23 +157,6 @@ def get_python_package_versions(package_name: str):
if __name__ == "__main__":
import argparse
logging.basicConfig(format="%(asctime)-15s %(message)s")
LOG.setLevel(logging.INFO)
parser = argparse.ArgumentParser(sys.argv[0])
parser.add_argument("--registry", action="store", default=REGISTRY)
parser.add_argument(
"--checkrebuild",
action="store_true",
default=False,
help="Rebuild images in case of outdated debian or python package",
)
parser.add_argument(
"--push", action="store_true", default=False, help="push images"
)
args = parser.parse_args()
REGISTRY = args.registry
cubicweb_versions = get_python_package_versions("cubicweb")
images = []
......@@ -299,9 +169,4 @@ if __name__ == "__main__":
images.append(CubicWebImage("buster", strict_version))
images.append(CubicWebImage("buster", "dev"))
if args.push:
push(images=images, last_debian_dist="buster")
else:
build(debian_dists=["buster"], images=images, rebuild=args.checkrebuild)
tag_aliases(images=images, last_debian_dist="buster")
generate_gitlab_ci(images)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment