build.py 10.6 KB
Newer Older
Philippe Pepiot's avatar
Philippe Pepiot committed
1
2
3
4
5
6
#!/usr/bin/env python3

import itertools
import subprocess
import sys
import logging
7
from dataclasses import dataclass
8
from typing import Optional, List, Any
9
10
11
12
from itertools import product


LOG = logging.getLogger(__name__)
Noé Gaumont's avatar
Noé Gaumont committed
13
14
REGISTRY = "logilab/cubicweb"
CWREPO = "https://forge.extranet.logilab.fr/cubicweb/cubicweb/"
15

16
17
18
19
20
21

@dataclass
class CubicWebImage:
    python: str
    debian: str
    package_name: str
22
23
24
25
    package_version: str

    @property
    def cubicweb_version(self):
Noé Gaumont's avatar
Noé Gaumont committed
26
        return self.package_version.split("-", 1)[0]
27
28
29

    @property
    def cubicweb_major_version(self):
Noé Gaumont's avatar
Noé Gaumont committed
30
        return self.cubicweb_version.rsplit(".", 1)[0]
31

Noé Gaumont's avatar
Noé Gaumont committed
32
    def __le__(self, other):
33
34
        lversion = [int(v) for v in self.cubicweb_version.split(".")]
        rversion = [int(v) for v in other.cubicweb_version.split(".")]
Carine Dengler's avatar
Carine Dengler committed
35
36
        return lversion <= rversion

Noé Gaumont's avatar
Noé Gaumont committed
37
38
39
    def __gt__(self, other):
        return not self <= other

40
41
    @property
    def tag(self):
Noé Gaumont's avatar
Noé Gaumont committed
42
43
        if self.package_version == "dev":
            return "dev"
44
45
46
47
48
        return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_version}"

    @property
    def base_docker_image(self):
        return f"{self.python}-{self.debian}"
Philippe Pepiot's avatar
Philippe Pepiot committed
49

Carine Dengler's avatar
Carine Dengler committed
50
51
    @property
    def major_tag(self):
Noé Gaumont's avatar
Noé Gaumont committed
52
53
        if self.package_version == "dev":
            return "dev"
Carine Dengler's avatar
Carine Dengler committed
54
55
        return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_major_version}"

Philippe Pepiot's avatar
Philippe Pepiot committed
56

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
def conditionnal_append(
    true_tab: List[Any], false_tab: List[Any], condition: bool, value: Any
):
    if condition:
        true_tab.append(value)
    else:
        false_tab.append(value)


def print_summary(action_type: str, succeed: List[Any], failed: List[Any]):
    LOG.info(f"########### {action_type.upper()} SUMMARY #############")
    LOG.info(f"Succeed {action_type.lower()}:")
    for elem in succeed:
        LOG.info(f"  -✅ {elem}")
    if failed:
        LOG.warning(f"Failed {action_type.lower()}:")
        for elem in failed:
            LOG.warning(red(f"  -❌ {elem}"))
    else:
        LOG.info(f"All {action_type.lower()} succeed. Good Job !")


Philippe Pepiot's avatar
Philippe Pepiot committed
79
def run(*args):
Noé Gaumont's avatar
Noé Gaumont committed
80
    LOG.info("%s", " ".join(args))
81
82
83
84
85
86
    # Capture both stdout and stder
    result = subprocess.run(args, capture_output=True)
    if result.returncode != 0:
        LOG.warning(result.stdout.decode())
        LOG.warning(red(result.stderr.decode()))
    return result
Philippe Pepiot's avatar
Philippe Pepiot committed
87
88
89


def check_call(*args):
Noé Gaumont's avatar
Noé Gaumont committed
90
    LOG.info("%s", " ".join(args))
Philippe Pepiot's avatar
Philippe Pepiot committed
91
92
93
94
    return subprocess.check_call(args)


def check_output(*args):
Noé Gaumont's avatar
Noé Gaumont committed
95
    LOG.info("%s", " ".join(args))
Philippe Pepiot's avatar
Philippe Pepiot committed
96
97
98
99
100
    return subprocess.check_output(args).decode().strip()


def _cwdev(_cache={}):
    if not _cache:
Noé Gaumont's avatar
Noé Gaumont committed
101
        rev = check_output("hg", "id", "-r", "default", CWREPO)
102
        # url anchor is used to invalidate docker cache.
Noé Gaumont's avatar
Noé Gaumont committed
103
104
105
106
107
        _cache[
            None
        ] = "{0}-/archive/branch/default/cubicweb-branch-default.tar.gz#rev={1}".format(
            CWREPO, rev
        )
Philippe Pepiot's avatar
Philippe Pepiot committed
108
109
    return _cache[None]

Noé Gaumont's avatar
Noé Gaumont committed
110

Noé Gaumont's avatar
Noé Gaumont committed
111
def get_major_tags(images: List[CubicWebImage]):
Carine Dengler's avatar
Carine Dengler committed
112
    latest = {}
113
    for cwimage in images:
Noé Gaumont's avatar
Noé Gaumont committed
114
        if cwimage.package_version == "dev":
115
116
            continue

Carine Dengler's avatar
Carine Dengler committed
117
118
119
        version = cwimage.cubicweb_major_version
        if version not in latest:
            latest[version] = cwimage
120
121
        elif latest[version] <= cwimage:
            latest[version] = cwimage
Carine Dengler's avatar
Carine Dengler committed
122
    last_major = max(latest.values())
Noé Gaumont's avatar
Noé Gaumont committed
123
    latest["latest"] = last_major
Noé Gaumont's avatar
Noé Gaumont committed
124
125
    return latest

Noé Gaumont's avatar
Noé Gaumont committed
126

Noé Gaumont's avatar
Noé Gaumont committed
127
128
def tag_aliases(images: List[CubicWebImage], last_debian_dist: str):
    latest = get_major_tags(images)
Carine Dengler's avatar
Carine Dengler committed
129

130
131
    tags = []
    fail_tags = []
Noé Gaumont's avatar
Noé Gaumont committed
132
    for major, img in latest.items():
Carine Dengler's avatar
Carine Dengler committed
133
        tag = f"{REGISTRY}:{major}"
Noé Gaumont's avatar
Noé Gaumont committed
134
135
        for onbuild in [None, "onbuild"]:
            if onbuild == "onbuild":
Carine Dengler's avatar
Carine Dengler committed
136
                tag = f"{REGISTRY}:{major}-{onbuild}"
Philippe Pepiot's avatar
Philippe Pepiot committed
137
            else:
Carine Dengler's avatar
Carine Dengler committed
138
139
                tag = f"{REGISTRY}:{major}"
            src = img.tag
140
141
142
143
            res = run("docker", "tag", src, tag)
            conditionnal_append(tags, fail_tags, res.returncode == 0, tag)
            res = run("docker", "tag", src, img.major_tag)
            conditionnal_append(tags, fail_tags, res.returncode == 0, img.major_tag)
Noé Gaumont's avatar
Noé Gaumont committed
144

145
    res = run(
Noé Gaumont's avatar
Noé Gaumont committed
146
147
148
149
150
        "docker",
        "tag",
        f"{REGISTRY}:{last_debian_dist}-buildpackage",
        f"{REGISTRY}:buildpackage",
    )
151
152
153
    conditionnal_append(
        tags, fail_tags, res.returncode == 0, f"{REGISTRY}:buildpackage"
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
154

155
    print_summary("tag", tags, fail_tags)
Philippe Pepiot's avatar
Philippe Pepiot committed
156

157
158

def build_image(image: CubicWebImage, onbuild: Optional[str], no_cache=False) -> bool:
Philippe Pepiot's avatar
Philippe Pepiot committed
159
    args = {}
Noé Gaumont's avatar
Noé Gaumont committed
160
    dockerfile = "Dockerfile"
Philippe Pepiot's avatar
Philippe Pepiot committed
161
    if onbuild is not None:
Noé Gaumont's avatar
Noé Gaumont committed
162
163
164
        dockerfile = "Dockerfile.onbuild"
        args["FROM"] = image.tag
        tag = image.tag + "-onbuild"
165
    else:
Noé Gaumont's avatar
Noé Gaumont committed
166
167
168
169
        args["PYTHON"] = "python" if image.python == "py27" else "python3"
        dockerfile = "Dockerfile"
        args["DIST"] = image.debian
        args["PACKAGES"] = f"{image.package_name}={image.package_version}"
170
        tag = image.tag
Noé Gaumont's avatar
Noé Gaumont committed
171
172
173
174
        if image.package_version == "dev":
            args["SOURCE_TARBALL"] = _cwdev()
            args["COMPONENT"] = "main"
            args["PACKAGES"] = f"{image.package_name}"
175
        else:
Noé Gaumont's avatar
Noé Gaumont committed
176
            args["COMPONENT"] = "cubicweb-{}".format(image.cubicweb_major_version)
177

Philippe Pepiot's avatar
Philippe Pepiot committed
178
    cmd = [
Noé Gaumont's avatar
Noé Gaumont committed
179
180
181
182
183
184
        "docker",
        "build",
        "-t",
        tag,
        "-f",
        dockerfile,
Philippe Pepiot's avatar
Philippe Pepiot committed
185
186
    ]
    if no_cache:
Noé Gaumont's avatar
Noé Gaumont committed
187
        cmd += ["--no-cache"]
Philippe Pepiot's avatar
Philippe Pepiot committed
188
    for key, value in args.items():
Noé Gaumont's avatar
Noé Gaumont committed
189
190
        cmd += ["--build-arg", "{}={}".format(key, value)]
    cmd += ["."]
191
192
193
194
    result = run(*cmd)
    if result.returncode != 0:
        return False
    return True
Philippe Pepiot's avatar
Philippe Pepiot committed
195
196
197


def green(text):
Noé Gaumont's avatar
Noé Gaumont committed
198
    return "\033[0;32m" + text + "\033[0m"
Philippe Pepiot's avatar
Philippe Pepiot committed
199
200
201


def red(text):
Noé Gaumont's avatar
Noé Gaumont committed
202
    return "\033[0;31m" + text + "\033[0m"
Philippe Pepiot's avatar
Philippe Pepiot committed
203
204


205
def build_buildpackage(dist):
Noé Gaumont's avatar
Noé Gaumont committed
206
207
208
209
210
211
212
213
214
215
216
217
218
    tag = "{}:{}-buildpackage".format(REGISTRY, dist)
    check_call(
        "docker",
        "build",
        "-t",
        tag,
        "--build-arg",
        "DIST={}".format(dist),
        "-f",
        "Dockerfile.buildpackage",
        ".",
    )

219

Noé Gaumont's avatar
Noé Gaumont committed
220
def get_cubicweb_images(debian_dists: List[str], python_versions: List[str]):
221
    images = []
222
    for dist, python_version in product(debian_dists, python_versions):
Noé Gaumont's avatar
Noé Gaumont committed
223
        python = "python" if python_version == "py27" else "python3"
224
        check_call(
Noé Gaumont's avatar
Noé Gaumont committed
225
226
227
228
229
230
231
232
233
234
235
            "docker",
            "build",
            ".",
            "-f",
            "Dockerfile.getversion",
            "-t",
            "cubicweb:getversion",
            "--build-arg",
            f"DIST={dist}",
            "--build-arg",
            f"PYTHON={python}",
236
237
        )
        versions = check_output(
Noé Gaumont's avatar
Noé Gaumont committed
238
            "docker", "run", "--rm", "cubicweb:getversion"
239
240
241
        ).splitlines()

        # filter release candidate version
Noé Gaumont's avatar
Noé Gaumont committed
242
        valid_versions = [version for version in versions if "~rc" not in version]
243
244
245
246
        for version in valid_versions:
            image = CubicWebImage(
                python=python_version,
                debian=dist,
Noé Gaumont's avatar
Noé Gaumont committed
247
248
                package_name=f"{python}-cubicweb",
                package_version=version,
Carine Dengler's avatar
Carine Dengler committed
249
            )
250
            images.append(image)
251
252
    return images

Noé Gaumont's avatar
Noé Gaumont committed
253
254

def build(debian_dists: List[str], images: List[CubicWebImage] = [], rebuild=False):
Carine Dengler's avatar
Carine Dengler committed
255
256
257
    for dist in debian_dists:
        if rebuild:
            # pull base images
Noé Gaumont's avatar
Noé Gaumont committed
258
            check_call("docker", "pull", f"debian:{dist}-slim")
Carine Dengler's avatar
Carine Dengler committed
259
        build_buildpackage(dist)
260
261
    built_images = []
    failed_images = []
262
    for image in images:
Noé Gaumont's avatar
Noé Gaumont committed
263
        for image_type in (None, "onbuild"):
264
265
            built = build_image(image, image_type)
            conditionnal_append(built_images, failed_images, built, image)
266

Carine Dengler's avatar
Carine Dengler committed
267
268
269
            if rebuild and image_type is None:
                tag = image.tag
                out = run(
Noé Gaumont's avatar
Noé Gaumont committed
270
271
272
273
274
275
276
277
278
279
280
                    "docker",
                    "run",
                    "--rm",
                    "-t",
                    "--user",
                    "root",
                    "--entrypoint",
                    "check-docker-updates.sh",
                    tag,
                    "apt",
                )
Carine Dengler's avatar
Carine Dengler committed
281
                if out.returncode == 1:
Noé Gaumont's avatar
Noé Gaumont committed
282
283
284
285
286
                    LOG.warning(
                        red("%s debian packages updates are available: %s"),
                        tag,
                        out.stdout,
                    )
Carine Dengler's avatar
Carine Dengler committed
287
288
                    build_image(image, image_type, no_cache=True)
                else:
Noé Gaumont's avatar
Noé Gaumont committed
289
290
                    assert (out.returncode, out.stdout) == (0, b""), out
                    LOG.info(green("%s debian packages are up-to-date"), tag)
Philippe Pepiot's avatar
Philippe Pepiot committed
291

292
293
    print_summary("build image", built_images, failed_images)

Philippe Pepiot's avatar
Philippe Pepiot committed
294

Noé Gaumont's avatar
Noé Gaumont committed
295
296
def push(images: List[CubicWebImage], last_debian_dist: str):
    latest = get_major_tags(images)
297
298
    succeed_push = []
    fail_push = []
Noé Gaumont's avatar
Noé Gaumont committed
299
    for image in images:
300
301
        res = run("docker", "push", image.tag)
        conditionnal_append(succeed_push, fail_push, res.returncode == 0, image.tag)
Noé Gaumont's avatar
Noé Gaumont committed
302
303
304

    for major, img in latest.items():
        tag = f"{REGISTRY}:{major}"
Noé Gaumont's avatar
Noé Gaumont committed
305
306
        for onbuild in [None, "onbuild"]:
            if onbuild == "onbuild":
Noé Gaumont's avatar
Noé Gaumont committed
307
                tag = f"{REGISTRY}:{major}-{onbuild}"
Philippe Pepiot's avatar
Philippe Pepiot committed
308
            else:
Noé Gaumont's avatar
Noé Gaumont committed
309
310
                tag = f"{REGISTRY}:{major}"
            src = img.tag
311
312
313
314
315
316
            res = run("docker", "push", major)
            conditionnal_append(succeed_push, fail_push, res.returncode == 0, major)
            res = run("docker", "push", img.major_tag)
            conditionnal_append(
                succeed_push, fail_push, res.returncode == 0, image.major_tag
            )
Noé Gaumont's avatar
Noé Gaumont committed
317

318
    res = run(
Noé Gaumont's avatar
Noé Gaumont committed
319
320
321
322
323
        "docker",
        "push",
        f"{REGISTRY}:{last_debian_dist}-buildpackage",
        f"{REGISTRY}:buildpackage",
    )
324
325
326
327
328
    conditionnal_append(
        succeed_push, fail_push, res.returncode == 0, f"{REGISTRY}:buildpackage"
    )

    print_summary("push image", succeed_push, fail_push)
Philippe Pepiot's avatar
Philippe Pepiot committed
329
330


Noé Gaumont's avatar
Noé Gaumont committed
331
if __name__ == "__main__":
Philippe Pepiot's avatar
Philippe Pepiot committed
332
    import argparse
Noé Gaumont's avatar
Noé Gaumont committed
333
334

    logging.basicConfig(format="%(asctime)-15s %(message)s")
Philippe Pepiot's avatar
Philippe Pepiot committed
335
336
    LOG.setLevel(logging.INFO)
    parser = argparse.ArgumentParser(sys.argv[0])
Noé Gaumont's avatar
Noé Gaumont committed
337
    parser.add_argument("--registry", action="store", default=REGISTRY)
Philippe Pepiot's avatar
Philippe Pepiot committed
338
    parser.add_argument(
Noé Gaumont's avatar
Noé Gaumont committed
339
340
341
342
343
        "--checkrebuild",
        action="store_true",
        default=False,
        help="Rebuild images in case of outdated debian or python package",
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
344
    parser.add_argument(
Noé Gaumont's avatar
Noé Gaumont committed
345
346
        "--push", action="store_true", default=False, help="push images"
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
347
348
    args = parser.parse_args()
    REGISTRY = args.registry
349
350
351
    images = get_cubicweb_images(["stretch", "buster"], ["py27"])
    images += get_cubicweb_images(["stretch"], ["py35"])
    images += get_cubicweb_images(["buster"], ["py37"])
Noé Gaumont's avatar
Noé Gaumont committed
352
    images.append(CubicWebImage("py37", "buster", "python3-cubicweb", "dev"))
Philippe Pepiot's avatar
Philippe Pepiot committed
353
    if args.push:
Noé Gaumont's avatar
Noé Gaumont committed
354
        push(images=images, last_debian_dist="buster")
Philippe Pepiot's avatar
Philippe Pepiot committed
355
    else:
Noé Gaumont's avatar
Noé Gaumont committed
356
357
358
        build(
            debian_dists=["stretch", "buster"], images=images, rebuild=args.checkrebuild
        )
Noé Gaumont's avatar
Noé Gaumont committed
359
        tag_aliases(images=images, last_debian_dist="buster")