build.py 8.29 KB
Newer Older
Philippe Pepiot's avatar
Philippe Pepiot committed
1
2
3
4
5
6
#!/usr/bin/env python3

import itertools
import subprocess
import sys
import logging
7
from dataclasses import dataclass
Noé Gaumont's avatar
Noé Gaumont committed
8
from typing import Optional, List
9
10
11
12
13
14
15
from itertools import product


LOG = logging.getLogger(__name__)
REGISTRY = 'logilab/cubicweb'
CWREPO = 'https://forge.extranet.logilab.fr/cubicweb/cubicweb/'

16
17
18
19
20
21

@dataclass
class CubicWebImage:
    python: str
    debian: str
    package_name: str
22
23
24
25
26
27
28
29
30
31
    package_version: str

    @property
    def cubicweb_version(self):
        return self.package_version.split('-', 1)[0]

    @property
    def cubicweb_major_version(self):
        return self.cubicweb_version.rsplit('.', 1)[0]

Noé Gaumont's avatar
Noé Gaumont committed
32
    def __le__(self, other):
33
34
        lversion = [int(v) for v in self.cubicweb_version.split(".")]
        rversion = [int(v) for v in other.cubicweb_version.split(".")]
Carine Dengler's avatar
Carine Dengler committed
35
36
        return lversion <= rversion

37

Noé Gaumont's avatar
Noé Gaumont committed
38
39
40
    def __gt__(self, other):
        return not self <= other

41
42
    @property
    def tag(self):
Carine Dengler's avatar
Carine Dengler committed
43
44
        if self.package_version == 'dev':
            return 'dev'
45
46
47
48
49
        return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_version}"

    @property
    def base_docker_image(self):
        return f"{self.python}-{self.debian}"
Philippe Pepiot's avatar
Philippe Pepiot committed
50

Carine Dengler's avatar
Carine Dengler committed
51
52
53
54
55
56
    @property
    def major_tag(self):
        if self.package_version == 'dev':
            return 'dev'
        return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_major_version}"

Philippe Pepiot's avatar
Philippe Pepiot committed
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76


def run(*args):
    LOG.info('%s', ' '.join(args))
    return subprocess.run(args, stdout=subprocess.PIPE)


def check_call(*args):
    LOG.info('%s', ' '.join(args))
    return subprocess.check_call(args)


def check_output(*args):
    LOG.info('%s', ' '.join(args))
    return subprocess.check_output(args).decode().strip()


def _cwdev(_cache={}):
    if not _cache:
        rev = check_output('hg', 'id', '-r', 'default', CWREPO)
77
78
        # url anchor is used to invalidate docker cache.
        _cache[None] = '{0}-/archive/branch/default/cubicweb-branch-default.tar.gz#rev={1}'.format(
Philippe Pepiot's avatar
Philippe Pepiot committed
79
80
81
            CWREPO, rev)
    return _cache[None]

Noé Gaumont's avatar
Noé Gaumont committed
82
def get_major_tags(images: List[CubicWebImage]):
Carine Dengler's avatar
Carine Dengler committed
83
    latest = {}
84
85
86
87
    for cwimage in images:
        if cwimage.package_version == 'dev':
            continue

Carine Dengler's avatar
Carine Dengler committed
88
89
90
        version = cwimage.cubicweb_major_version
        if version not in latest:
            latest[version] = cwimage
91
92
        elif latest[version] <= cwimage:
            latest[version] = cwimage
Carine Dengler's avatar
Carine Dengler committed
93
    last_major = max(latest.values())
Noé Gaumont's avatar
Noé Gaumont committed
94
95
96
97
98
    latest['latest'] = last_major
    return latest

def tag_aliases(images: List[CubicWebImage], last_debian_dist: str):
    latest = get_major_tags(images)
Carine Dengler's avatar
Carine Dengler committed
99

Noé Gaumont's avatar
Noé Gaumont committed
100
    for major, img in latest.items():
Carine Dengler's avatar
Carine Dengler committed
101
        tag = f"{REGISTRY}:{major}"
Philippe Pepiot's avatar
Philippe Pepiot committed
102
        for onbuild in [None, 'onbuild']:
Carine Dengler's avatar
Carine Dengler committed
103
104
            if onbuild == 'onbuild':
                tag = f"{REGISTRY}:{major}-{onbuild}"
Philippe Pepiot's avatar
Philippe Pepiot committed
105
            else:
Carine Dengler's avatar
Carine Dengler committed
106
107
                tag = f"{REGISTRY}:{major}"
            src = img.tag
Philippe Pepiot's avatar
Philippe Pepiot committed
108
            check_call('docker', 'tag', src, tag)
Carine Dengler's avatar
Carine Dengler committed
109
            check_call('docker', 'tag', src, img.major_tag)
Noé Gaumont's avatar
Noé Gaumont committed
110
111

    check_call('docker', 'tag', f'{REGISTRY}:{last_debian_dist}-buildpackage', f"{REGISTRY}:buildpackage")
Philippe Pepiot's avatar
Philippe Pepiot committed
112
113


114
def build_image(image: CubicWebImage, onbuild: Optional[str], no_cache=False):
Philippe Pepiot's avatar
Philippe Pepiot committed
115
116
117
118
    args = {}
    dockerfile = 'Dockerfile'
    if onbuild is not None:
        dockerfile = 'Dockerfile.onbuild'
119
120
121
122
123
124
        args['FROM'] = image.tag
        tag = image.tag + '-onbuild'
    else:
        args['PYTHON'] = 'python' if image.python == 'py27' else 'python3'
        dockerfile = 'Dockerfile'
        args['DIST'] = image.debian
Carine Dengler's avatar
Carine Dengler committed
125
        args['PACKAGES'] = f"{image.package_name}={image.package_version}"
126
127
        tag = image.tag
        if image.package_version == 'dev':
128
129
            args['SOURCE_TARBALL'] = _cwdev()
            args['COMPONENT'] = 'main'
Noé Gaumont's avatar
Noé Gaumont committed
130
            args['PACKAGES'] = f"{image.package_name}"
131
        else:
132
133
            args['COMPONENT'] = 'cubicweb-{}'.format(image.cubicweb_major_version)

Philippe Pepiot's avatar
Philippe Pepiot committed
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
    cmd = [
        'docker', 'build', '-t', tag,
        '-f', dockerfile,
    ]
    if no_cache:
        cmd += ['--no-cache']
    for key, value in args.items():
        cmd += ['--build-arg', '{}={}'.format(key, value)]
    cmd += ['.']
    check_call(*cmd)


def green(text):
    return '\033[0;32m' + text + '\033[0m'


def red(text):
    return '\033[0;31m' + text + '\033[0m'


154
155
156
157
158
159
def build_buildpackage(dist):
    tag = '{}:{}-buildpackage'.format(REGISTRY, dist)
    check_call('docker', 'build', '-t', tag,
               '--build-arg', 'DIST={}'.format(dist),
               '-f', 'Dockerfile.buildpackage', '.')

Noé Gaumont's avatar
Noé Gaumont committed
160
def get_cubicweb_images(debian_dists: List[str], python_versions: List[str]):
161
    images = []
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
    for dist, python_version in product(debian_dists, python_versions):
        python = 'python' if python_version == 'py27' else 'python3'
        check_call(
            'docker',
            'build',
            '.',
            '-f',
            'Dockerfile.getversion',
            '-t',
            'cubicweb:getversion',
            '--build-arg',
            f'DIST={dist}',
            '--build-arg',
            f"PYTHON={python}"
        )
        versions = check_output(
            'docker', 'run', '--rm', 'cubicweb:getversion'
        ).splitlines()

        # filter release candidate version
        valid_versions = [
            version for version in versions if '~rc' not in version
        ]
        for version in valid_versions:
            image = CubicWebImage(
                python=python_version,
                debian=dist,
                package_name=f'{python}-cubicweb',
                package_version=version
Carine Dengler's avatar
Carine Dengler committed
191
            )
192
            images.append(image)
193
194
    return images

Noé Gaumont's avatar
Noé Gaumont committed
195
def build(debian_dists: List[str], images: List[CubicWebImage]=[], rebuild=False):
Carine Dengler's avatar
Carine Dengler committed
196
197
198
199
200
    for dist in debian_dists:
        if rebuild:
            # pull base images
            check_call('docker', 'pull', f'debian:{dist}-slim')
        build_buildpackage(dist)
201
202
203
204
    for image in images:
        for image_type in (None, 'onbuild'):
            build_image(image, image_type)

Carine Dengler's avatar
Carine Dengler committed
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
            if rebuild and image_type is None:
                tag = image.tag
                out = run(
                    'docker', 'run', '--rm', '-t',
                    '--user', 'root',
                    '--entrypoint', 'check-docker-updates.sh',
                    tag, 'apt')
                if out.returncode == 1:
                    LOG.warning(red(
                        '%s debian packages updates are available: %s'),
                        tag, out.stdout)
                    build_image(image, image_type, no_cache=True)
                else:
                    assert (out.returncode, out.stdout) == (0, b''), out
                    LOG.info(green('%s debian packages are up-to-date'), tag)
Philippe Pepiot's avatar
Philippe Pepiot committed
220
221


Noé Gaumont's avatar
Noé Gaumont committed
222
223
224
225
226
227
228
def push(images: List[CubicWebImage], last_debian_dist: str):
    latest = get_major_tags(images)
    for image in images:
        check_call('docker', 'push', image.tag)

    for major, img in latest.items():
        tag = f"{REGISTRY}:{major}"
Philippe Pepiot's avatar
Philippe Pepiot committed
229
        for onbuild in [None, 'onbuild']:
Noé Gaumont's avatar
Noé Gaumont committed
230
231
            if onbuild == 'onbuild':
                tag = f"{REGISTRY}:{major}-{onbuild}"
Philippe Pepiot's avatar
Philippe Pepiot committed
232
            else:
Noé Gaumont's avatar
Noé Gaumont committed
233
234
235
236
237
238
                tag = f"{REGISTRY}:{major}"
            src = img.tag
            check_call('docker', 'push', major)
            check_call('docker', 'push', img.major_tag)

    check_call('docker', 'push', f'{REGISTRY}:{last_debian_dist}-buildpackage', f"{REGISTRY}:buildpackage")
Philippe Pepiot's avatar
Philippe Pepiot committed
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254


if __name__ == '__main__':
    import argparse
    logging.basicConfig(format='%(asctime)-15s %(message)s')
    LOG.setLevel(logging.INFO)
    parser = argparse.ArgumentParser(sys.argv[0])
    parser.add_argument('--registry', action='store', default=REGISTRY)
    parser.add_argument(
        '--checkrebuild', action='store_true', default=False,
        help='Rebuild images in case of outdated debian or python package')
    parser.add_argument(
        '--push', action='store_true', default=False,
        help='push images')
    args = parser.parse_args()
    REGISTRY = args.registry
255
256
257
    images = get_cubicweb_images(["stretch", "buster"], ["py27"])
    images += get_cubicweb_images(["stretch"], ["py35"])
    images += get_cubicweb_images(["buster"], ["py37"])
Carine Dengler's avatar
Carine Dengler committed
258
259
260
    images.append(
        CubicWebImage("py37", "buster", "python3-cubicweb", "dev")
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
261
    if args.push:
Noé Gaumont's avatar
Noé Gaumont committed
262
        push(images=images, last_debian_dist="buster")
Philippe Pepiot's avatar
Philippe Pepiot committed
263
    else:
Noé Gaumont's avatar
Noé Gaumont committed
264
265
        build(debian_dists=["stretch", "buster"], images=images, rebuild=args.checkrebuild)
        tag_aliases(images=images, last_debian_dist="buster")