#!/usr/bin/env python3 import itertools import subprocess import sys import logging from dataclasses import dataclass from typing import Optional, List, Any from itertools import product LOG = logging.getLogger(__name__) REGISTRY = "logilab/cubicweb" CWREPO = "https://forge.extranet.logilab.fr/cubicweb/cubicweb/" @dataclass class CubicWebImage: python: str debian: str package_name: str package_version: str @property def cubicweb_version(self): return self.package_version.split("-", 1)[0] @property def cubicweb_major_version(self): return self.cubicweb_version.rsplit(".", 1)[0] def __le__(self, other): lversion = [int(v) for v in self.cubicweb_version.split(".")] rversion = [int(v) for v in other.cubicweb_version.split(".")] return lversion <= rversion def __gt__(self, other): return not self <= other @property def tag(self): if self.package_version == "dev": return f"{REGISTRY}:dev" return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_version}" @property def base_docker_image(self): return f"{self.python}-{self.debian}" @property def major_tag(self): if self.package_version == "dev": return f"{REGISTRY}:dev" return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_major_version}" def conditionnal_append( true_tab: List[Any], false_tab: List[Any], condition: bool, value: Any ): if condition: true_tab.append(value) else: false_tab.append(value) def print_summary(action_type: str, succeed: List[Any], failed: List[Any]): LOG.info(f"########### {action_type.upper()} SUMMARY #############") LOG.info(f"Succeed {action_type.lower()}:") for elem in succeed: LOG.info(f" -✅ {elem}") if failed: LOG.warning(f"Failed {action_type.lower()}:") for elem in failed: LOG.warning(red(f" -❌ {elem}")) else: LOG.info(f"All {action_type.lower()} succeed. Good Job !") def run(*args): LOG.info("%s", " ".join(args)) # Capture both stdout and stder result = subprocess.run(args, capture_output=True) if result.returncode != 0: LOG.warning(result.stdout.decode()) LOG.warning(red(result.stderr.decode())) return result def check_call(*args): LOG.info("%s", " ".join(args)) return subprocess.check_call(args) def check_output(*args): LOG.info("%s", " ".join(args)) return subprocess.check_output(args).decode().strip() def _cwdev(_cache={}): if not _cache: rev = check_output("hg", "id", "-r", "default", CWREPO) # url anchor is used to invalidate docker cache. _cache[ None ] = "{0}-/archive/branch/default/cubicweb-branch-default.tar.gz#rev={1}".format( CWREPO, rev ) return _cache[None] def get_major_tags(images: List[CubicWebImage]): latest = {} for cwimage in images: if cwimage.package_version == "dev": continue version = cwimage.cubicweb_major_version if version not in latest: latest[version] = cwimage elif latest[version] <= cwimage: latest[version] = cwimage last_major = max(latest.values()) latest["latest"] = last_major return latest def tag_aliases(images: List[CubicWebImage], last_debian_dist: str): latest = get_major_tags(images) tags = [] fail_tags = [] for major, img in latest.items(): tag = f"{REGISTRY}:{major}" for onbuild in [None, "onbuild"]: if onbuild == "onbuild": tag = f"{REGISTRY}:{major}-{onbuild}" else: tag = f"{REGISTRY}:{major}" src = img.tag res = run("docker", "tag", src, tag) conditionnal_append(tags, fail_tags, res.returncode == 0, tag) res = run("docker", "tag", src, img.major_tag) conditionnal_append(tags, fail_tags, res.returncode == 0, img.major_tag) res = run( "docker", "tag", f"{REGISTRY}:{last_debian_dist}-buildpackage", f"{REGISTRY}:buildpackage", ) conditionnal_append( tags, fail_tags, res.returncode == 0, f"{REGISTRY}:buildpackage" ) print_summary("tag", tags, fail_tags) def build_image(image: CubicWebImage, onbuild: Optional[str], no_cache=False) -> bool: args = {} dockerfile = "Dockerfile" if onbuild is not None: dockerfile = "Dockerfile.onbuild" args["FROM"] = image.tag tag = image.tag + "-onbuild" else: args["PYTHON"] = "python" if image.python == "py27" else "python3" dockerfile = "Dockerfile" args["DIST"] = image.debian args["PACKAGES"] = f"{image.package_name}={image.package_version}" tag = image.tag if image.package_version == "dev": args["SOURCE_TARBALL"] = _cwdev() args["COMPONENT"] = "main" args["PACKAGES"] = f"{image.package_name}" else: args["COMPONENT"] = "cubicweb-{}".format(image.cubicweb_major_version) cmd = [ "docker", "build", "-t", tag, "-f", dockerfile, ] if no_cache: cmd += ["--no-cache"] for key, value in args.items(): cmd += ["--build-arg", "{}={}".format(key, value)] cmd += ["."] result = run(*cmd) if result.returncode != 0: return False return True def green(text): return "\033[0;32m" + text + "\033[0m" def red(text): return "\033[0;31m" + text + "\033[0m" def build_buildpackage(dist): tag = "{}:{}-buildpackage".format(REGISTRY, dist) check_call( "docker", "build", "-t", tag, "--build-arg", "DIST={}".format(dist), "-f", "Dockerfile.buildpackage", ".", ) def get_cubicweb_images(debian_dists: List[str], python_versions: List[str]): images = [] for dist, python_version in product(debian_dists, python_versions): python = "python" if python_version == "py27" else "python3" check_call( "docker", "build", ".", "-f", "Dockerfile.getversion", "-t", "cubicweb:getversion", "--build-arg", f"DIST={dist}", "--build-arg", f"PYTHON={python}", ) versions = check_output( "docker", "run", "--rm", "cubicweb:getversion" ).splitlines() # filter release candidate version valid_versions = [version for version in versions if "~rc" not in version] for version in valid_versions: image = CubicWebImage( python=python_version, debian=dist, package_name=f"{python}-cubicweb", package_version=version, ) images.append(image) return images def build(debian_dists: List[str], images: List[CubicWebImage] = [], rebuild=False): for dist in debian_dists: if rebuild: # pull base images check_call("docker", "pull", f"debian:{dist}-slim") build_buildpackage(dist) built_images = [] failed_images = [] for image in images: for image_type in (None, "onbuild"): built = build_image(image, image_type) conditionnal_append(built_images, failed_images, built, f"{image}-{image_type}") # Do no try to update image not built if built and rebuild and image_type is None: tag = image.tag out = run( "docker", "run", "--rm", "-t", "--user", "root", "--entrypoint", "check-docker-updates.sh", tag, "apt", ) if out.returncode == 1: LOG.warning( red("%s debian packages updates are available: %s"), tag, out.stdout, ) build_image(image, image_type, no_cache=True) else: assert (out.returncode, out.stdout) == (0, b""), out LOG.info(green("%s debian packages are up-to-date"), tag) print_summary("build image", built_images, failed_images) def push(images: List[CubicWebImage], last_debian_dist: str): latest = get_major_tags(images) succeed_push = [] fail_push = [] for image in images: res = run("docker", "push", image.tag) conditionnal_append(succeed_push, fail_push, res.returncode == 0, image.tag) for major, img in latest.items(): tag = f"{REGISTRY}:{major}" for onbuild in [None, "onbuild"]: if onbuild == "onbuild": tag = f"{REGISTRY}:{major}-{onbuild}" else: tag = f"{REGISTRY}:{major}" res = run("docker", "push", tag) conditionnal_append(succeed_push, fail_push, res.returncode == 0, tag) res = run("docker", "push", img.major_tag) conditionnal_append( succeed_push, fail_push, res.returncode == 0, image.major_tag ) res = run( "docker", "push", f"{REGISTRY}:{last_debian_dist}-buildpackage", f"{REGISTRY}:buildpackage", ) conditionnal_append( succeed_push, fail_push, res.returncode == 0, f"{REGISTRY}:buildpackage" ) print_summary("push image", succeed_push, fail_push) if __name__ == "__main__": import argparse logging.basicConfig(format="%(asctime)-15s %(message)s") LOG.setLevel(logging.INFO) parser = argparse.ArgumentParser(sys.argv[0]) parser.add_argument("--registry", action="store", default=REGISTRY) parser.add_argument( "--checkrebuild", action="store_true", default=False, help="Rebuild images in case of outdated debian or python package", ) parser.add_argument( "--push", action="store_true", default=False, help="push images" ) args = parser.parse_args() REGISTRY = args.registry images = get_cubicweb_images(["stretch", "buster"], ["py27"]) images += get_cubicweb_images(["stretch"], ["py35"]) images += get_cubicweb_images(["buster"], ["py37"]) images.append(CubicWebImage("py37", "buster", "python3-cubicweb", "dev")) if args.push: push(images=images, last_debian_dist="buster") else: build( debian_dists=["stretch", "buster"], images=images, rebuild=args.checkrebuild ) tag_aliases(images=images, last_debian_dist="buster")