build.py 8.89 KB
Newer Older
Philippe Pepiot's avatar
Philippe Pepiot committed
1
2
3
4
5
6
#!/usr/bin/env python3

import itertools
import subprocess
import sys
import logging
7
from dataclasses import dataclass
Noé Gaumont's avatar
Noé Gaumont committed
8
from typing import Optional, List
9
10
11
12
13
14

@dataclass
class CubicWebImage:
    python: str
    debian: str
    package_name: str
15
16
17
18
19
20
21
22
23
24
    package_version: str

    @property
    def cubicweb_version(self):
        return self.package_version.split('-', 1)[0]

    @property
    def cubicweb_major_version(self):
        return self.cubicweb_version.rsplit('.', 1)[0]

Noé Gaumont's avatar
Noé Gaumont committed
25
    def __le__(self, other):
Carine Dengler's avatar
Carine Dengler committed
26
27
28
29
        lversion = self.cubicweb_version.split(".")
        rversion = other.cubicweb_version.split(".")
        return lversion <= rversion

Noé Gaumont's avatar
Noé Gaumont committed
30
31
32
    def __gt__(self, other):
        return not self <= other

33
34
    @property
    def tag(self):
Carine Dengler's avatar
Carine Dengler committed
35
36
        if self.package_version == 'dev':
            return 'dev'
37
38
39
40
41
        return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_version}"

    @property
    def base_docker_image(self):
        return f"{self.python}-{self.debian}"
Philippe Pepiot's avatar
Philippe Pepiot committed
42

Carine Dengler's avatar
Carine Dengler committed
43
44
45
46
47
48
    @property
    def major_tag(self):
        if self.package_version == 'dev':
            return 'dev'
        return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_major_version}"

Philippe Pepiot's avatar
Philippe Pepiot committed
49
50

LOG = logging.getLogger(__name__)
51
REGISTRY = 'logilab/cubicweb'
52
CWREPO = 'https://forge.extranet.logilab.fr/cubicweb/cubicweb/'
Philippe Pepiot's avatar
Philippe Pepiot committed
53
MATRIX = [
Noé Gaumont's avatar
Noé Gaumont committed
54
55
56
57
    # (['py27'], ['stretch', 'buster'], [None, '3.25', '3.26'],
    #     [None, 'onbuild']),
    # (['py35'], ['stretch'], [None, '3.26', '3.27'], [None, 'onbuild']),
    (['py37'], ['buster'], ['3.29'], [None, 'onbuild']),
Philippe Pepiot's avatar
Philippe Pepiot committed
58
59
]
ALIASES = (
Noé Gaumont's avatar
Noé Gaumont committed
60
61
62
63
    # ('3.25', 'py27-buster-3.25'),
    # ('3.26', 'py37-buster-3.26'),
    # ('3.27', 'py37-buster-3.27'),
    # ('3.28', 'py37-buster-3.28'),
64
    ('3.29', 'py37-buster-3.29'),
Noé Gaumont's avatar
Noé Gaumont committed
65
66
    # ('dev', 'py37-buster-dev'),
    # ('latest', 'py37-buster-3.29'),
67
    ('buildpackage', 'buster-buildpackage'),
Philippe Pepiot's avatar
Philippe Pepiot committed
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
)


def run(*args):
    LOG.info('%s', ' '.join(args))
    return subprocess.run(args, stdout=subprocess.PIPE)


def check_call(*args):
    LOG.info('%s', ' '.join(args))
    return subprocess.check_call(args)


def check_output(*args):
    LOG.info('%s', ' '.join(args))
    return subprocess.check_output(args).decode().strip()


def _cwdev(_cache={}):
    if not _cache:
        rev = check_output('hg', 'id', '-r', 'default', CWREPO)
89
90
        # url anchor is used to invalidate docker cache.
        _cache[None] = '{0}-/archive/branch/default/cubicweb-branch-default.tar.gz#rev={1}'.format(
Philippe Pepiot's avatar
Philippe Pepiot committed
91
92
93
            CWREPO, rev)
    return _cache[None]

Noé Gaumont's avatar
Noé Gaumont committed
94
def get_major_tags(images: List[CubicWebImage]):
Carine Dengler's avatar
Carine Dengler committed
95
96
97
98
99
100
101
102
103
    latest = {}
    for cwimage in [img for img in images if img.package_version != 'dev']:
        version = cwimage.cubicweb_major_version
        if version not in latest:
            latest[version] = cwimage
        else:
            if latest[version] <= cwimage:
                latest[version] = cwimage
    last_major = max(latest.values())
Noé Gaumont's avatar
Noé Gaumont committed
104
105
106
107
108
    latest['latest'] = last_major
    return latest

def tag_aliases(images: List[CubicWebImage], last_debian_dist: str):
    latest = get_major_tags(images)
Carine Dengler's avatar
Carine Dengler committed
109

Noé Gaumont's avatar
Noé Gaumont committed
110
    for major, img in latest.items():
Carine Dengler's avatar
Carine Dengler committed
111
        tag = f"{REGISTRY}:{major}"
Philippe Pepiot's avatar
Philippe Pepiot committed
112
        for onbuild in [None, 'onbuild']:
Carine Dengler's avatar
Carine Dengler committed
113
114
            if onbuild == 'onbuild':
                tag = f"{REGISTRY}:{major}-{onbuild}"
Philippe Pepiot's avatar
Philippe Pepiot committed
115
            else:
Carine Dengler's avatar
Carine Dengler committed
116
117
                tag = f"{REGISTRY}:{major}"
            src = img.tag
Philippe Pepiot's avatar
Philippe Pepiot committed
118
            check_call('docker', 'tag', src, tag)
Carine Dengler's avatar
Carine Dengler committed
119
            check_call('docker', 'tag', src, img.major_tag)
Noé Gaumont's avatar
Noé Gaumont committed
120
121

    check_call('docker', 'tag', f'{REGISTRY}:{last_debian_dist}-buildpackage', f"{REGISTRY}:buildpackage")
Philippe Pepiot's avatar
Philippe Pepiot committed
122
123


124
def build_image(image: CubicWebImage, onbuild: Optional[str], no_cache=False, registry=None):
Philippe Pepiot's avatar
Philippe Pepiot committed
125
126
127
128
    args = {}
    dockerfile = 'Dockerfile'
    if onbuild is not None:
        dockerfile = 'Dockerfile.onbuild'
129
130
131
132
133
134
        args['FROM'] = image.tag
        tag = image.tag + '-onbuild'
    else:
        args['PYTHON'] = 'python' if image.python == 'py27' else 'python3'
        dockerfile = 'Dockerfile'
        args['DIST'] = image.debian
Carine Dengler's avatar
Carine Dengler committed
135
        args['PACKAGES'] = f"{image.package_name}={image.package_version}"
136
137
        tag = image.tag
        if image.package_version == 'dev':
138
139
            args['SOURCE_TARBALL'] = _cwdev()
            args['COMPONENT'] = 'main'
Noé Gaumont's avatar
Noé Gaumont committed
140
            args['PACKAGES'] = f"{image.package_name}"
141
        else:
142
143
            args['COMPONENT'] = 'cubicweb-{}'.format(image.cubicweb_major_version)

Philippe Pepiot's avatar
Philippe Pepiot committed
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
    cmd = [
        'docker', 'build', '-t', tag,
        '-f', dockerfile,
    ]
    if no_cache:
        cmd += ['--no-cache']
    for key, value in args.items():
        cmd += ['--build-arg', '{}={}'.format(key, value)]
    cmd += ['.']
    check_call(*cmd)


def green(text):
    return '\033[0;32m' + text + '\033[0m'


def red(text):
    return '\033[0;31m' + text + '\033[0m'


164
165
166
167
168
169
def build_buildpackage(dist):
    tag = '{}:{}-buildpackage'.format(REGISTRY, dist)
    check_call('docker', 'build', '-t', tag,
               '--build-arg', 'DIST={}'.format(dist),
               '-f', 'Dockerfile.buildpackage', '.')

Noé Gaumont's avatar
Noé Gaumont committed
170
def get_cubicweb_images(debian_dists: List[str], python_versions: List[str]):
171
    images = []
Carine Dengler's avatar
Carine Dengler committed
172
    for dist in debian_dists:
173
174
175
176
177
178
179
180
181
182
183
184
185
186
        for python_version in python_versions:
            python = 'python' if python_version == 'py27' else 'python3'
            check_call(
                'docker',
                'build',
                '.',
                '-f',
                'Dockerfile.getversion',
                '-t',
                'cubicweb:getversion',
                '--build-arg',
                f'DIST={dist}',
                '--build-arg',
                f"PYTHON={python}"
Carine Dengler's avatar
Carine Dengler committed
187
            )
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
            versions = check_output(
                'docker', 'run', '--rm', 'cubicweb:getversion'
            ).splitlines()

            # filter release candidate version
            valid_versions = [
                version for version in versions if '~rc' not in version
            ]
            for version in valid_versions:
                image = CubicWebImage(
                    python=python_version,
                    debian=dist,
                    package_name=f'{python}-cubicweb',
                    package_version=version
                )
                images.append(image)
204
205
    return images

Noé Gaumont's avatar
Noé Gaumont committed
206
def build(debian_dists: List[str], images: List[CubicWebImage]=[], rebuild=False):
Carine Dengler's avatar
Carine Dengler committed
207
208
209
210
211
    for dist in debian_dists:
        if rebuild:
            # pull base images
            check_call('docker', 'pull', f'debian:{dist}-slim')
        build_buildpackage(dist)
212
213
214
215
    for image in images:
        for image_type in (None, 'onbuild'):
            build_image(image, image_type)

Carine Dengler's avatar
Carine Dengler committed
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
            if rebuild and image_type is None:
                tag = image.tag
                out = run(
                    'docker', 'run', '--rm', '-t',
                    '--user', 'root',
                    '--entrypoint', 'check-docker-updates.sh',
                    tag, 'apt')
                if out.returncode == 1:
                    LOG.warning(red(
                        '%s debian packages updates are available: %s'),
                        tag, out.stdout)
                    build_image(image, image_type, no_cache=True)
                else:
                    assert (out.returncode, out.stdout) == (0, b''), out
                    LOG.info(green('%s debian packages are up-to-date'), tag)
Philippe Pepiot's avatar
Philippe Pepiot committed
231
232


Noé Gaumont's avatar
Noé Gaumont committed
233
234
235
236
237
238
239
def push(images: List[CubicWebImage], last_debian_dist: str):
    latest = get_major_tags(images)
    for image in images:
        check_call('docker', 'push', image.tag)

    for major, img in latest.items():
        tag = f"{REGISTRY}:{major}"
Philippe Pepiot's avatar
Philippe Pepiot committed
240
        for onbuild in [None, 'onbuild']:
Noé Gaumont's avatar
Noé Gaumont committed
241
242
            if onbuild == 'onbuild':
                tag = f"{REGISTRY}:{major}-{onbuild}"
Philippe Pepiot's avatar
Philippe Pepiot committed
243
            else:
Noé Gaumont's avatar
Noé Gaumont committed
244
245
246
247
248
249
                tag = f"{REGISTRY}:{major}"
            src = img.tag
            check_call('docker', 'push', major)
            check_call('docker', 'push', img.major_tag)

    check_call('docker', 'push', f'{REGISTRY}:{last_debian_dist}-buildpackage', f"{REGISTRY}:buildpackage")
Philippe Pepiot's avatar
Philippe Pepiot committed
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265


if __name__ == '__main__':
    import argparse
    logging.basicConfig(format='%(asctime)-15s %(message)s')
    LOG.setLevel(logging.INFO)
    parser = argparse.ArgumentParser(sys.argv[0])
    parser.add_argument('--registry', action='store', default=REGISTRY)
    parser.add_argument(
        '--checkrebuild', action='store_true', default=False,
        help='Rebuild images in case of outdated debian or python package')
    parser.add_argument(
        '--push', action='store_true', default=False,
        help='push images')
    args = parser.parse_args()
    REGISTRY = args.registry
266
267
268
    images = get_cubicweb_images(["stretch", "buster"], ["py27"])
    images += get_cubicweb_images(["stretch"], ["py35"])
    images += get_cubicweb_images(["buster"], ["py37"])
Carine Dengler's avatar
Carine Dengler committed
269
270
271
    images.append(
        CubicWebImage("py37", "buster", "python3-cubicweb", "dev")
    )
Philippe Pepiot's avatar
Philippe Pepiot committed
272
    if args.push:
Noé Gaumont's avatar
Noé Gaumont committed
273
        push(images=images, last_debian_dist="buster")
Philippe Pepiot's avatar
Philippe Pepiot committed
274
    else:
Noé Gaumont's avatar
Noé Gaumont committed
275
276
        build(debian_dists=["stretch", "buster"], images=images, rebuild=args.checkrebuild)
        tag_aliases(images=images, last_debian_dist="buster")