build.py 8.67 KB
Newer Older
Philippe Pepiot's avatar
Philippe Pepiot committed
1
2
3
4
5
6
#!/usr/bin/env python3

import itertools
import subprocess
import sys
import logging
7
from dataclasses import dataclass
Noé Gaumont's avatar
Noé Gaumont committed
8
from typing import Optional, List
9
10
11
12
from itertools import product


LOG = logging.getLogger(__name__)
Noé Gaumont's avatar
Noé Gaumont committed
13
14
REGISTRY = "logilab/cubicweb"
CWREPO = "https://forge.extranet.logilab.fr/cubicweb/cubicweb/"
15

16
17
18
19
20
21

@dataclass
class CubicWebImage:
    python: str
    debian: str
    package_name: str
22
23
24
25
    package_version: str

    @property
    def cubicweb_version(self):
Noé Gaumont's avatar
Noé Gaumont committed
26
        return self.package_version.split("-", 1)[0]
27
28
29

    @property
    def cubicweb_major_version(self):
Noé Gaumont's avatar
Noé Gaumont committed
30
        return self.cubicweb_version.rsplit(".", 1)[0]
31

Noé Gaumont's avatar
Noé Gaumont committed
32
    def __le__(self, other):
33
34
        lversion = [int(v) for v in self.cubicweb_version.split(".")]
        rversion = [int(v) for v in other.cubicweb_version.split(".")]
Carine Dengler's avatar
Carine Dengler committed
35
36
        return lversion <= rversion

Noé Gaumont's avatar
Noé Gaumont committed
37
38
39
    def __gt__(self, other):
        return not self <= other

40
41
    @property
    def tag(self):
Noé Gaumont's avatar
Noé Gaumont committed
42
43
        if self.package_version == "dev":
            return "dev"
44
45
46
47
48
        return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_version}"

    @property
    def base_docker_image(self):
        return f"{self.python}-{self.debian}"
Philippe Pepiot's avatar
Philippe Pepiot committed
49

Carine Dengler's avatar
Carine Dengler committed
50
51
    @property
    def major_tag(self):
Noé Gaumont's avatar
Noé Gaumont committed
52
53
        if self.package_version == "dev":
            return "dev"
Carine Dengler's avatar
Carine Dengler committed
54
55
        return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_major_version}"

Philippe Pepiot's avatar
Philippe Pepiot committed
56
57

def run(*args):
Noé Gaumont's avatar
Noé Gaumont committed
58
    LOG.info("%s", " ".join(args))
Philippe Pepiot's avatar
Philippe Pepiot committed
59
60
61
62
    return subprocess.run(args, stdout=subprocess.PIPE)


def check_call(*args):
Noé Gaumont's avatar
Noé Gaumont committed
63
    LOG.info("%s", " ".join(args))
Philippe Pepiot's avatar
Philippe Pepiot committed
64
65
66
67
    return subprocess.check_call(args)


def check_output(*args):
Noé Gaumont's avatar
Noé Gaumont committed
68
    LOG.info("%s", " ".join(args))
Philippe Pepiot's avatar
Philippe Pepiot committed
69
70
71
72
73
    return subprocess.check_output(args).decode().strip()


def _cwdev(_cache={}):
    if not _cache:
Noé Gaumont's avatar
Noé Gaumont committed
74
        rev = check_output("hg", "id", "-r", "default", CWREPO)
75
        # url anchor is used to invalidate docker cache.
Noé Gaumont's avatar
Noé Gaumont committed
76
77
78
79
80
        _cache[
            None
        ] = "{0}-/archive/branch/default/cubicweb-branch-default.tar.gz#rev={1}".format(
            CWREPO, rev
        )
Philippe Pepiot's avatar
Philippe Pepiot committed
81
82
    return _cache[None]

Noé Gaumont's avatar
Noé Gaumont committed
83

Noé Gaumont's avatar
Noé Gaumont committed
84
def get_major_tags(images: List[CubicWebImage]):
Carine Dengler's avatar
Carine Dengler committed
85
    latest = {}
86
    for cwimage in images:
Noé Gaumont's avatar
Noé Gaumont committed
87
        if cwimage.package_version == "dev":
88
89
            continue

Carine Dengler's avatar
Carine Dengler committed
90
91
92
        version = cwimage.cubicweb_major_version
        if version not in latest:
            latest[version] = cwimage
93
94
        elif latest[version] <= cwimage:
            latest[version] = cwimage
Carine Dengler's avatar
Carine Dengler committed
95
    last_major = max(latest.values())
Noé Gaumont's avatar
Noé Gaumont committed
96
    latest["latest"] = last_major
Noé Gaumont's avatar
Noé Gaumont committed
97
98
    return latest

Noé Gaumont's avatar
Noé Gaumont committed
99

Noé Gaumont's avatar
Noé Gaumont committed
100
101
def tag_aliases(images: List[CubicWebImage], last_debian_dist: str):
    latest = get_major_tags(images)
Carine Dengler's avatar
Carine Dengler committed
102

Noé Gaumont's avatar
Noé Gaumont committed
103
    for major, img in latest.items():
Carine Dengler's avatar
Carine Dengler committed
104
        tag = f"{REGISTRY}:{major}"
Noé Gaumont's avatar
Noé Gaumont committed
105
106
        for onbuild in [None, "onbuild"]:
            if onbuild == "onbuild":
Carine Dengler's avatar
Carine Dengler committed
107
                tag = f"{REGISTRY}:{major}-{onbuild}"
Philippe Pepiot's avatar
Philippe Pepiot committed
108
            else:
Carine Dengler's avatar
Carine Dengler committed
109
110
                tag = f"{REGISTRY}:{major}"
            src = img.tag
Noé Gaumont's avatar
Noé Gaumont committed
111
112
113
114
115
116
117
118
119
            check_call("docker", "tag", src, tag)
            check_call("docker", "tag", src, img.major_tag)

    check_call(
        "docker",
        "tag",
        f"{REGISTRY}:{last_debian_dist}-buildpackage",
        f"{REGISTRY}:buildpackage",
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
120
121


122
def build_image(image: CubicWebImage, onbuild: Optional[str], no_cache=False):
Philippe Pepiot's avatar
Philippe Pepiot committed
123
    args = {}
Noé Gaumont's avatar
Noé Gaumont committed
124
    dockerfile = "Dockerfile"
Philippe Pepiot's avatar
Philippe Pepiot committed
125
    if onbuild is not None:
Noé Gaumont's avatar
Noé Gaumont committed
126
127
128
        dockerfile = "Dockerfile.onbuild"
        args["FROM"] = image.tag
        tag = image.tag + "-onbuild"
129
    else:
Noé Gaumont's avatar
Noé Gaumont committed
130
131
132
133
        args["PYTHON"] = "python" if image.python == "py27" else "python3"
        dockerfile = "Dockerfile"
        args["DIST"] = image.debian
        args["PACKAGES"] = f"{image.package_name}={image.package_version}"
134
        tag = image.tag
Noé Gaumont's avatar
Noé Gaumont committed
135
136
137
138
        if image.package_version == "dev":
            args["SOURCE_TARBALL"] = _cwdev()
            args["COMPONENT"] = "main"
            args["PACKAGES"] = f"{image.package_name}"
139
        else:
Noé Gaumont's avatar
Noé Gaumont committed
140
            args["COMPONENT"] = "cubicweb-{}".format(image.cubicweb_major_version)
141

Philippe Pepiot's avatar
Philippe Pepiot committed
142
    cmd = [
Noé Gaumont's avatar
Noé Gaumont committed
143
144
145
146
147
148
        "docker",
        "build",
        "-t",
        tag,
        "-f",
        dockerfile,
Philippe Pepiot's avatar
Philippe Pepiot committed
149
150
    ]
    if no_cache:
Noé Gaumont's avatar
Noé Gaumont committed
151
        cmd += ["--no-cache"]
Philippe Pepiot's avatar
Philippe Pepiot committed
152
    for key, value in args.items():
Noé Gaumont's avatar
Noé Gaumont committed
153
154
        cmd += ["--build-arg", "{}={}".format(key, value)]
    cmd += ["."]
Philippe Pepiot's avatar
Philippe Pepiot committed
155
156
157
158
    check_call(*cmd)


def green(text):
Noé Gaumont's avatar
Noé Gaumont committed
159
    return "\033[0;32m" + text + "\033[0m"
Philippe Pepiot's avatar
Philippe Pepiot committed
160
161
162


def red(text):
Noé Gaumont's avatar
Noé Gaumont committed
163
    return "\033[0;31m" + text + "\033[0m"
Philippe Pepiot's avatar
Philippe Pepiot committed
164
165


166
def build_buildpackage(dist):
Noé Gaumont's avatar
Noé Gaumont committed
167
168
169
170
171
172
173
174
175
176
177
178
179
    tag = "{}:{}-buildpackage".format(REGISTRY, dist)
    check_call(
        "docker",
        "build",
        "-t",
        tag,
        "--build-arg",
        "DIST={}".format(dist),
        "-f",
        "Dockerfile.buildpackage",
        ".",
    )

180

Noé Gaumont's avatar
Noé Gaumont committed
181
def get_cubicweb_images(debian_dists: List[str], python_versions: List[str]):
182
    images = []
183
    for dist, python_version in product(debian_dists, python_versions):
Noé Gaumont's avatar
Noé Gaumont committed
184
        python = "python" if python_version == "py27" else "python3"
185
        check_call(
Noé Gaumont's avatar
Noé Gaumont committed
186
187
188
189
190
191
192
193
194
195
196
            "docker",
            "build",
            ".",
            "-f",
            "Dockerfile.getversion",
            "-t",
            "cubicweb:getversion",
            "--build-arg",
            f"DIST={dist}",
            "--build-arg",
            f"PYTHON={python}",
197
198
        )
        versions = check_output(
Noé Gaumont's avatar
Noé Gaumont committed
199
            "docker", "run", "--rm", "cubicweb:getversion"
200
201
202
        ).splitlines()

        # filter release candidate version
Noé Gaumont's avatar
Noé Gaumont committed
203
        valid_versions = [version for version in versions if "~rc" not in version]
204
205
206
207
        for version in valid_versions:
            image = CubicWebImage(
                python=python_version,
                debian=dist,
Noé Gaumont's avatar
Noé Gaumont committed
208
209
                package_name=f"{python}-cubicweb",
                package_version=version,
Carine Dengler's avatar
Carine Dengler committed
210
            )
211
            images.append(image)
212
213
    return images

Noé Gaumont's avatar
Noé Gaumont committed
214
215

def build(debian_dists: List[str], images: List[CubicWebImage] = [], rebuild=False):
Carine Dengler's avatar
Carine Dengler committed
216
217
218
    for dist in debian_dists:
        if rebuild:
            # pull base images
Noé Gaumont's avatar
Noé Gaumont committed
219
            check_call("docker", "pull", f"debian:{dist}-slim")
Carine Dengler's avatar
Carine Dengler committed
220
        build_buildpackage(dist)
221
    for image in images:
Noé Gaumont's avatar
Noé Gaumont committed
222
        for image_type in (None, "onbuild"):
223
224
            build_image(image, image_type)

Carine Dengler's avatar
Carine Dengler committed
225
226
227
            if rebuild and image_type is None:
                tag = image.tag
                out = run(
Noé Gaumont's avatar
Noé Gaumont committed
228
229
230
231
232
233
234
235
236
237
238
                    "docker",
                    "run",
                    "--rm",
                    "-t",
                    "--user",
                    "root",
                    "--entrypoint",
                    "check-docker-updates.sh",
                    tag,
                    "apt",
                )
Carine Dengler's avatar
Carine Dengler committed
239
                if out.returncode == 1:
Noé Gaumont's avatar
Noé Gaumont committed
240
241
242
243
244
                    LOG.warning(
                        red("%s debian packages updates are available: %s"),
                        tag,
                        out.stdout,
                    )
Carine Dengler's avatar
Carine Dengler committed
245
246
                    build_image(image, image_type, no_cache=True)
                else:
Noé Gaumont's avatar
Noé Gaumont committed
247
248
                    assert (out.returncode, out.stdout) == (0, b""), out
                    LOG.info(green("%s debian packages are up-to-date"), tag)
Philippe Pepiot's avatar
Philippe Pepiot committed
249
250


Noé Gaumont's avatar
Noé Gaumont committed
251
252
253
def push(images: List[CubicWebImage], last_debian_dist: str):
    latest = get_major_tags(images)
    for image in images:
Noé Gaumont's avatar
Noé Gaumont committed
254
        check_call("docker", "push", image.tag)
Noé Gaumont's avatar
Noé Gaumont committed
255
256
257

    for major, img in latest.items():
        tag = f"{REGISTRY}:{major}"
Noé Gaumont's avatar
Noé Gaumont committed
258
259
        for onbuild in [None, "onbuild"]:
            if onbuild == "onbuild":
Noé Gaumont's avatar
Noé Gaumont committed
260
                tag = f"{REGISTRY}:{major}-{onbuild}"
Philippe Pepiot's avatar
Philippe Pepiot committed
261
            else:
Noé Gaumont's avatar
Noé Gaumont committed
262
263
                tag = f"{REGISTRY}:{major}"
            src = img.tag
Noé Gaumont's avatar
Noé Gaumont committed
264
265
266
267
268
269
270
271
272
            check_call("docker", "push", major)
            check_call("docker", "push", img.major_tag)

    check_call(
        "docker",
        "push",
        f"{REGISTRY}:{last_debian_dist}-buildpackage",
        f"{REGISTRY}:buildpackage",
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
273
274


Noé Gaumont's avatar
Noé Gaumont committed
275
if __name__ == "__main__":
Philippe Pepiot's avatar
Philippe Pepiot committed
276
    import argparse
Noé Gaumont's avatar
Noé Gaumont committed
277
278

    logging.basicConfig(format="%(asctime)-15s %(message)s")
Philippe Pepiot's avatar
Philippe Pepiot committed
279
280
    LOG.setLevel(logging.INFO)
    parser = argparse.ArgumentParser(sys.argv[0])
Noé Gaumont's avatar
Noé Gaumont committed
281
    parser.add_argument("--registry", action="store", default=REGISTRY)
Philippe Pepiot's avatar
Philippe Pepiot committed
282
    parser.add_argument(
Noé Gaumont's avatar
Noé Gaumont committed
283
284
285
286
287
        "--checkrebuild",
        action="store_true",
        default=False,
        help="Rebuild images in case of outdated debian or python package",
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
288
    parser.add_argument(
Noé Gaumont's avatar
Noé Gaumont committed
289
290
        "--push", action="store_true", default=False, help="push images"
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
291
292
    args = parser.parse_args()
    REGISTRY = args.registry
293
294
295
    images = get_cubicweb_images(["stretch", "buster"], ["py27"])
    images += get_cubicweb_images(["stretch"], ["py35"])
    images += get_cubicweb_images(["buster"], ["py37"])
Noé Gaumont's avatar
Noé Gaumont committed
296
    images.append(CubicWebImage("py37", "buster", "python3-cubicweb", "dev"))
Philippe Pepiot's avatar
Philippe Pepiot committed
297
    if args.push:
Noé Gaumont's avatar
Noé Gaumont committed
298
        push(images=images, last_debian_dist="buster")
Philippe Pepiot's avatar
Philippe Pepiot committed
299
    else:
Noé Gaumont's avatar
Noé Gaumont committed
300
301
302
        build(
            debian_dists=["stretch", "buster"], images=images, rebuild=args.checkrebuild
        )
Noé Gaumont's avatar
Noé Gaumont committed
303
        tag_aliases(images=images, last_debian_dist="buster")