build.py 9.23 KB
Newer Older
Philippe Pepiot's avatar
Philippe Pepiot committed
1
2
3
4
5
6
#!/usr/bin/env python3

import itertools
import subprocess
import sys
import logging
7
from dataclasses import dataclass
8
9
10
11
from typing import Optional, List, Any, Union
import json
from urllib.request import urlopen
from distutils.version import StrictVersion
12
13
14


LOG = logging.getLogger(__name__)
Noé Gaumont's avatar
Noé Gaumont committed
15
16
REGISTRY = "logilab/cubicweb"
CWREPO = "https://forge.extranet.logilab.fr/cubicweb/cubicweb/"
17

18
19
20

@dataclass
class CubicWebImage:
21
22
    debian_dist: str
    cubicweb_version: Union[StrictVersion, str]
23
24
25

    @property
    def cubicweb_major_version(self):
26
        return ".".join(map(str, self.cubicweb_version.version[:2]))
27

Noé Gaumont's avatar
Noé Gaumont committed
28
    def __le__(self, other):
29
        return self.cubicweb_version <= other.cubicweb_version
Carine Dengler's avatar
Carine Dengler committed
30

Noé Gaumont's avatar
Noé Gaumont committed
31
32
33
    def __gt__(self, other):
        return not self <= other

34
35
36
    def is_dev(self):
        return isinstance(self.cubicweb_version, str) and self.cubicweb_version == "dev"

37
38
    @property
    def tag(self):
39
        if self.is_dev():
Noé Gaumont's avatar
Noé Gaumont committed
40
            return f"{REGISTRY}:dev"
41
        return f"{REGISTRY}:{self.debian_dist}-{self.cubicweb_version}"
Philippe Pepiot's avatar
Philippe Pepiot committed
42

Carine Dengler's avatar
Carine Dengler committed
43
44
    @property
    def major_tag(self):
45
        if self.is_dev():
Noé Gaumont's avatar
Noé Gaumont committed
46
            return f"{REGISTRY}:dev"
47
        return f"{REGISTRY}:{self.debian_dist}-{self.cubicweb_major_version}"
Carine Dengler's avatar
Carine Dengler committed
48

Philippe Pepiot's avatar
Philippe Pepiot committed
49

50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
def conditionnal_append(
    true_tab: List[Any], false_tab: List[Any], condition: bool, value: Any
):
    if condition:
        true_tab.append(value)
    else:
        false_tab.append(value)


def print_summary(action_type: str, succeed: List[Any], failed: List[Any]):
    LOG.info(f"########### {action_type.upper()} SUMMARY #############")
    LOG.info(f"Succeed {action_type.lower()}:")
    for elem in succeed:
        LOG.info(f"  -✅ {elem}")
    if failed:
        LOG.warning(f"Failed {action_type.lower()}:")
        for elem in failed:
            LOG.warning(red(f"  -❌ {elem}"))
    else:
        LOG.info(f"All {action_type.lower()} succeed. Good Job !")


Philippe Pepiot's avatar
Philippe Pepiot committed
72
def run(*args):
Noé Gaumont's avatar
Noé Gaumont committed
73
    LOG.info("%s", " ".join(args))
74
75
76
77
78
79
    # Capture both stdout and stder
    result = subprocess.run(args, capture_output=True)
    if result.returncode != 0:
        LOG.warning(result.stdout.decode())
        LOG.warning(red(result.stderr.decode()))
    return result
Philippe Pepiot's avatar
Philippe Pepiot committed
80
81
82


def check_call(*args):
Noé Gaumont's avatar
Noé Gaumont committed
83
    LOG.info("%s", " ".join(args))
Philippe Pepiot's avatar
Philippe Pepiot committed
84
85
86
87
    return subprocess.check_call(args)


def check_output(*args):
Noé Gaumont's avatar
Noé Gaumont committed
88
    LOG.info("%s", " ".join(args))
Philippe Pepiot's avatar
Philippe Pepiot committed
89
90
91
92
93
    return subprocess.check_output(args).decode().strip()


def _cwdev(_cache={}):
    if not _cache:
Noé Gaumont's avatar
Noé Gaumont committed
94
        rev = check_output("hg", "id", "-r", "default", CWREPO)
95
        # url anchor is used to invalidate docker cache.
Noé Gaumont's avatar
Noé Gaumont committed
96
97
98
99
100
        _cache[
            None
        ] = "{0}-/archive/branch/default/cubicweb-branch-default.tar.gz#rev={1}".format(
            CWREPO, rev
        )
Philippe Pepiot's avatar
Philippe Pepiot committed
101
102
    return _cache[None]

Noé Gaumont's avatar
Noé Gaumont committed
103

Noé Gaumont's avatar
Noé Gaumont committed
104
def get_major_tags(images: List[CubicWebImage]):
Carine Dengler's avatar
Carine Dengler committed
105
    latest = {}
106
    for cwimage in images:
107
        if cwimage.is_dev():
108
109
            continue

Carine Dengler's avatar
Carine Dengler committed
110
111
112
        version = cwimage.cubicweb_major_version
        if version not in latest:
            latest[version] = cwimage
113
114
        elif latest[version] <= cwimage:
            latest[version] = cwimage
Carine Dengler's avatar
Carine Dengler committed
115
    last_major = max(latest.values())
Noé Gaumont's avatar
Noé Gaumont committed
116
    latest["latest"] = last_major
Noé Gaumont's avatar
Noé Gaumont committed
117
118
    return latest

Noé Gaumont's avatar
Noé Gaumont committed
119

Noé Gaumont's avatar
Noé Gaumont committed
120
121
def tag_aliases(images: List[CubicWebImage], last_debian_dist: str):
    latest = get_major_tags(images)
Carine Dengler's avatar
Carine Dengler committed
122

123
124
    tags = []
    fail_tags = []
Noé Gaumont's avatar
Noé Gaumont committed
125
    for major, img in latest.items():
Carine Dengler's avatar
Carine Dengler committed
126
        tag = f"{REGISTRY}:{major}"
Noé Gaumont's avatar
Noé Gaumont committed
127
128
        for onbuild in [None, "onbuild"]:
            if onbuild == "onbuild":
Carine Dengler's avatar
Carine Dengler committed
129
                tag = f"{REGISTRY}:{major}-{onbuild}"
Philippe Pepiot's avatar
Philippe Pepiot committed
130
            else:
Carine Dengler's avatar
Carine Dengler committed
131
132
                tag = f"{REGISTRY}:{major}"
            src = img.tag
133
134
135
136
            res = run("docker", "tag", src, tag)
            conditionnal_append(tags, fail_tags, res.returncode == 0, tag)
            res = run("docker", "tag", src, img.major_tag)
            conditionnal_append(tags, fail_tags, res.returncode == 0, img.major_tag)
Noé Gaumont's avatar
Noé Gaumont committed
137

138
    res = run(
Noé Gaumont's avatar
Noé Gaumont committed
139
140
141
142
143
        "docker",
        "tag",
        f"{REGISTRY}:{last_debian_dist}-buildpackage",
        f"{REGISTRY}:buildpackage",
    )
144
145
146
    conditionnal_append(
        tags, fail_tags, res.returncode == 0, f"{REGISTRY}:buildpackage"
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
147

148
    print_summary("tag", tags, fail_tags)
Philippe Pepiot's avatar
Philippe Pepiot committed
149

150
151

def build_image(image: CubicWebImage, onbuild: Optional[str], no_cache=False) -> bool:
Philippe Pepiot's avatar
Philippe Pepiot committed
152
    args = {}
Noé Gaumont's avatar
Noé Gaumont committed
153
    dockerfile = "Dockerfile"
Philippe Pepiot's avatar
Philippe Pepiot committed
154
    if onbuild is not None:
Noé Gaumont's avatar
Noé Gaumont committed
155
156
157
        dockerfile = "Dockerfile.onbuild"
        args["FROM"] = image.tag
        tag = image.tag + "-onbuild"
158
    else:
Noé Gaumont's avatar
Noé Gaumont committed
159
        dockerfile = "Dockerfile"
160
161
        args["DIST"] = image.debian_dist
        args["CUBICWEB_SOURCE"] = f"cubicweb[pyramid]=={image.cubicweb_version}"
162
        tag = image.tag
163
164
        if image.is_dev():
            args["CUBICWEB_SOURCE"] = _cwdev()
165

Philippe Pepiot's avatar
Philippe Pepiot committed
166
    cmd = [
Noé Gaumont's avatar
Noé Gaumont committed
167
168
169
170
171
172
        "docker",
        "build",
        "-t",
        tag,
        "-f",
        dockerfile,
Philippe Pepiot's avatar
Philippe Pepiot committed
173
174
    ]
    if no_cache:
Noé Gaumont's avatar
Noé Gaumont committed
175
        cmd += ["--no-cache"]
Philippe Pepiot's avatar
Philippe Pepiot committed
176
    for key, value in args.items():
Noé Gaumont's avatar
Noé Gaumont committed
177
178
        cmd += ["--build-arg", "{}={}".format(key, value)]
    cmd += ["."]
179
180
181
182
    result = run(*cmd)
    if result.returncode != 0:
        return False
    return True
Philippe Pepiot's avatar
Philippe Pepiot committed
183
184
185


def green(text):
Noé Gaumont's avatar
Noé Gaumont committed
186
    return "\033[0;32m" + text + "\033[0m"
Philippe Pepiot's avatar
Philippe Pepiot committed
187
188
189


def red(text):
Noé Gaumont's avatar
Noé Gaumont committed
190
    return "\033[0;31m" + text + "\033[0m"
Philippe Pepiot's avatar
Philippe Pepiot committed
191
192


193
def build_buildpackage(dist):
Noé Gaumont's avatar
Noé Gaumont committed
194
195
196
197
198
199
200
201
202
203
204
205
206
    tag = "{}:{}-buildpackage".format(REGISTRY, dist)
    check_call(
        "docker",
        "build",
        "-t",
        tag,
        "--build-arg",
        "DIST={}".format(dist),
        "-f",
        "Dockerfile.buildpackage",
        ".",
    )

207

Noé Gaumont's avatar
Noé Gaumont committed
208
def build(debian_dists: List[str], images: List[CubicWebImage] = [], rebuild=False):
Carine Dengler's avatar
Carine Dengler committed
209
210
211
    for dist in debian_dists:
        if rebuild:
            # pull base images
Noé Gaumont's avatar
Noé Gaumont committed
212
            check_call("docker", "pull", f"debian:{dist}-slim")
Carine Dengler's avatar
Carine Dengler committed
213
        build_buildpackage(dist)
214
215
    built_images = []
    failed_images = []
216
    for image in images:
Noé Gaumont's avatar
Noé Gaumont committed
217
        for image_type in (None, "onbuild"):
218
            built = build_image(image, image_type)
219
220
221
            conditionnal_append(
                built_images, failed_images, built, f"{image}-{image_type}"
            )
222

223
224
            # Do no try to update image not built
            if built and rebuild and image_type is None:
Carine Dengler's avatar
Carine Dengler committed
225
226
                tag = image.tag
                out = run(
Noé Gaumont's avatar
Noé Gaumont committed
227
228
229
230
231
232
233
234
235
236
237
                    "docker",
                    "run",
                    "--rm",
                    "-t",
                    "--user",
                    "root",
                    "--entrypoint",
                    "check-docker-updates.sh",
                    tag,
                    "apt",
                )
Carine Dengler's avatar
Carine Dengler committed
238
                if out.returncode == 1:
Noé Gaumont's avatar
Noé Gaumont committed
239
240
241
242
243
                    LOG.warning(
                        red("%s debian packages updates are available: %s"),
                        tag,
                        out.stdout,
                    )
Carine Dengler's avatar
Carine Dengler committed
244
245
                    build_image(image, image_type, no_cache=True)
                else:
Noé Gaumont's avatar
Noé Gaumont committed
246
247
                    assert (out.returncode, out.stdout) == (0, b""), out
                    LOG.info(green("%s debian packages are up-to-date"), tag)
Philippe Pepiot's avatar
Philippe Pepiot committed
248

249
250
    print_summary("build image", built_images, failed_images)

Philippe Pepiot's avatar
Philippe Pepiot committed
251

Noé Gaumont's avatar
Noé Gaumont committed
252
253
def push(images: List[CubicWebImage], last_debian_dist: str):
    latest = get_major_tags(images)
254
255
    succeed_push = []
    fail_push = []
Noé Gaumont's avatar
Noé Gaumont committed
256

257
    minor_tags = (image.tag for image in images)
Noé Gaumont's avatar
Noé Gaumont committed
258

259
260
261
262
    def major_tags_gen():
        for major, img in latest.items():
            yield f"{REGISTRY}:{major}-onbuild"
            yield f"{REGISTRY}:{major}"
263
            if major != "latest":
264
265
266
                yield img.major_tag

    buildpackage_tags = (
Noé Gaumont's avatar
Noé Gaumont committed
267
268
269
        f"{REGISTRY}:{last_debian_dist}-buildpackage",
        f"{REGISTRY}:buildpackage",
    )
270
271

    for tag in itertools.chain(minor_tags, major_tags_gen(), buildpackage_tags):
Noé Gaumont's avatar
Noé Gaumont committed
272
273
274
275
276
        res = run(
            "docker",
            "push",
            tag,
        )
277
        conditionnal_append(succeed_push, fail_push, res.returncode == 0, tag)
278
279

    print_summary("push image", succeed_push, fail_push)
Philippe Pepiot's avatar
Philippe Pepiot committed
280
281


282
283
284
285
286
287
288
289
290
291
def get_python_package_versions(package_name: str):
    """
    Retrieve all the package version available on pypi
    """
    url = f"https://pypi.org/pypi/{package_name}/json"
    data = json.load(urlopen(url))
    versions = data["releases"].keys()
    return versions


Noé Gaumont's avatar
Noé Gaumont committed
292
if __name__ == "__main__":
Philippe Pepiot's avatar
Philippe Pepiot committed
293
    import argparse
Noé Gaumont's avatar
Noé Gaumont committed
294
295

    logging.basicConfig(format="%(asctime)-15s %(message)s")
Philippe Pepiot's avatar
Philippe Pepiot committed
296
297
    LOG.setLevel(logging.INFO)
    parser = argparse.ArgumentParser(sys.argv[0])
Noé Gaumont's avatar
Noé Gaumont committed
298
    parser.add_argument("--registry", action="store", default=REGISTRY)
Philippe Pepiot's avatar
Philippe Pepiot committed
299
    parser.add_argument(
Noé Gaumont's avatar
Noé Gaumont committed
300
301
302
303
304
        "--checkrebuild",
        action="store_true",
        default=False,
        help="Rebuild images in case of outdated debian or python package",
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
305
    parser.add_argument(
Noé Gaumont's avatar
Noé Gaumont committed
306
307
        "--push", action="store_true", default=False, help="push images"
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
308
309
    args = parser.parse_args()
    REGISTRY = args.registry
310
311
312
313
314
315
316
317
318
319
320
321
322
    cubicweb_versions = get_python_package_versions("cubicweb")

    images = []
    for version in cubicweb_versions:
        if "rc" in version:
            continue
        strict_version = StrictVersion(version)
        if strict_version < StrictVersion("3.26.0"):
            continue
        images.append(CubicWebImage("buster", strict_version))

    images.append(CubicWebImage("buster", "dev"))

Philippe Pepiot's avatar
Philippe Pepiot committed
323
    if args.push:
Noé Gaumont's avatar
Noé Gaumont committed
324
        push(images=images, last_debian_dist="buster")
Philippe Pepiot's avatar
Philippe Pepiot committed
325
    else:
326
        build(debian_dists=["buster"], images=images, rebuild=args.checkrebuild)
Noé Gaumont's avatar
Noé Gaumont committed
327
        tag_aliases(images=images, last_debian_dist="buster")