Commit 43bcb6f0 authored by Noé Gaumont's avatar Noé Gaumont 🐙
Browse files

style: run black

parent f777462847b8
...@@ -10,8 +10,8 @@ from itertools import product ...@@ -10,8 +10,8 @@ from itertools import product
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
REGISTRY = 'logilab/cubicweb' REGISTRY = "logilab/cubicweb"
CWREPO = 'https://forge.extranet.logilab.fr/cubicweb/cubicweb/' CWREPO = "https://forge.extranet.logilab.fr/cubicweb/cubicweb/"
@dataclass @dataclass
...@@ -23,25 +23,24 @@ class CubicWebImage: ...@@ -23,25 +23,24 @@ class CubicWebImage:
@property @property
def cubicweb_version(self): def cubicweb_version(self):
return self.package_version.split('-', 1)[0] return self.package_version.split("-", 1)[0]
@property @property
def cubicweb_major_version(self): def cubicweb_major_version(self):
return self.cubicweb_version.rsplit('.', 1)[0] return self.cubicweb_version.rsplit(".", 1)[0]
def __le__(self, other): def __le__(self, other):
lversion = [int(v) for v in self.cubicweb_version.split(".")] lversion = [int(v) for v in self.cubicweb_version.split(".")]
rversion = [int(v) for v in other.cubicweb_version.split(".")] rversion = [int(v) for v in other.cubicweb_version.split(".")]
return lversion <= rversion return lversion <= rversion
def __gt__(self, other): def __gt__(self, other):
return not self <= other return not self <= other
@property @property
def tag(self): def tag(self):
if self.package_version == 'dev': if self.package_version == "dev":
return 'dev' return "dev"
return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_version}" return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_version}"
@property @property
...@@ -50,39 +49,42 @@ class CubicWebImage: ...@@ -50,39 +49,42 @@ class CubicWebImage:
@property @property
def major_tag(self): def major_tag(self):
if self.package_version == 'dev': if self.package_version == "dev":
return 'dev' return "dev"
return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_major_version}" return f"{REGISTRY}:{self.base_docker_image}-{self.cubicweb_major_version}"
def run(*args): def run(*args):
LOG.info('%s', ' '.join(args)) LOG.info("%s", " ".join(args))
return subprocess.run(args, stdout=subprocess.PIPE) return subprocess.run(args, stdout=subprocess.PIPE)
def check_call(*args): def check_call(*args):
LOG.info('%s', ' '.join(args)) LOG.info("%s", " ".join(args))
return subprocess.check_call(args) return subprocess.check_call(args)
def check_output(*args): def check_output(*args):
LOG.info('%s', ' '.join(args)) LOG.info("%s", " ".join(args))
return subprocess.check_output(args).decode().strip() return subprocess.check_output(args).decode().strip()
def _cwdev(_cache={}): def _cwdev(_cache={}):
if not _cache: if not _cache:
rev = check_output('hg', 'id', '-r', 'default', CWREPO) rev = check_output("hg", "id", "-r", "default", CWREPO)
# url anchor is used to invalidate docker cache. # url anchor is used to invalidate docker cache.
_cache[None] = '{0}-/archive/branch/default/cubicweb-branch-default.tar.gz#rev={1}'.format( _cache[
CWREPO, rev) None
] = "{0}-/archive/branch/default/cubicweb-branch-default.tar.gz#rev={1}".format(
CWREPO, rev
)
return _cache[None] return _cache[None]
def get_major_tags(images: List[CubicWebImage]): def get_major_tags(images: List[CubicWebImage]):
latest = {} latest = {}
for cwimage in images: for cwimage in images:
if cwimage.package_version == 'dev': if cwimage.package_version == "dev":
continue continue
version = cwimage.cubicweb_major_version version = cwimage.cubicweb_major_version
...@@ -91,175 +93,211 @@ def get_major_tags(images: List[CubicWebImage]): ...@@ -91,175 +93,211 @@ def get_major_tags(images: List[CubicWebImage]):
elif latest[version] <= cwimage: elif latest[version] <= cwimage:
latest[version] = cwimage latest[version] = cwimage
last_major = max(latest.values()) last_major = max(latest.values())
latest['latest'] = last_major latest["latest"] = last_major
return latest return latest
def tag_aliases(images: List[CubicWebImage], last_debian_dist: str): def tag_aliases(images: List[CubicWebImage], last_debian_dist: str):
latest = get_major_tags(images) latest = get_major_tags(images)
for major, img in latest.items(): for major, img in latest.items():
tag = f"{REGISTRY}:{major}" tag = f"{REGISTRY}:{major}"
for onbuild in [None, 'onbuild']: for onbuild in [None, "onbuild"]:
if onbuild == 'onbuild': if onbuild == "onbuild":
tag = f"{REGISTRY}:{major}-{onbuild}" tag = f"{REGISTRY}:{major}-{onbuild}"
else: else:
tag = f"{REGISTRY}:{major}" tag = f"{REGISTRY}:{major}"
src = img.tag src = img.tag
check_call('docker', 'tag', src, tag) check_call("docker", "tag", src, tag)
check_call('docker', 'tag', src, img.major_tag) check_call("docker", "tag", src, img.major_tag)
check_call('docker', 'tag', f'{REGISTRY}:{last_debian_dist}-buildpackage', f"{REGISTRY}:buildpackage") check_call(
"docker",
"tag",
f"{REGISTRY}:{last_debian_dist}-buildpackage",
f"{REGISTRY}:buildpackage",
)
def build_image(image: CubicWebImage, onbuild: Optional[str], no_cache=False): def build_image(image: CubicWebImage, onbuild: Optional[str], no_cache=False):
args = {} args = {}
dockerfile = 'Dockerfile' dockerfile = "Dockerfile"
if onbuild is not None: if onbuild is not None:
dockerfile = 'Dockerfile.onbuild' dockerfile = "Dockerfile.onbuild"
args['FROM'] = image.tag args["FROM"] = image.tag
tag = image.tag + '-onbuild' tag = image.tag + "-onbuild"
else: else:
args['PYTHON'] = 'python' if image.python == 'py27' else 'python3' args["PYTHON"] = "python" if image.python == "py27" else "python3"
dockerfile = 'Dockerfile' dockerfile = "Dockerfile"
args['DIST'] = image.debian args["DIST"] = image.debian
args['PACKAGES'] = f"{image.package_name}={image.package_version}" args["PACKAGES"] = f"{image.package_name}={image.package_version}"
tag = image.tag tag = image.tag
if image.package_version == 'dev': if image.package_version == "dev":
args['SOURCE_TARBALL'] = _cwdev() args["SOURCE_TARBALL"] = _cwdev()
args['COMPONENT'] = 'main' args["COMPONENT"] = "main"
args['PACKAGES'] = f"{image.package_name}" args["PACKAGES"] = f"{image.package_name}"
else: else:
args['COMPONENT'] = 'cubicweb-{}'.format(image.cubicweb_major_version) args["COMPONENT"] = "cubicweb-{}".format(image.cubicweb_major_version)
cmd = [ cmd = [
'docker', 'build', '-t', tag, "docker",
'-f', dockerfile, "build",
"-t",
tag,
"-f",
dockerfile,
] ]
if no_cache: if no_cache:
cmd += ['--no-cache'] cmd += ["--no-cache"]
for key, value in args.items(): for key, value in args.items():
cmd += ['--build-arg', '{}={}'.format(key, value)] cmd += ["--build-arg", "{}={}".format(key, value)]
cmd += ['.'] cmd += ["."]
check_call(*cmd) check_call(*cmd)
def green(text): def green(text):
return '\033[0;32m' + text + '\033[0m' return "\033[0;32m" + text + "\033[0m"
def red(text): def red(text):
return '\033[0;31m' + text + '\033[0m' return "\033[0;31m" + text + "\033[0m"
def build_buildpackage(dist): def build_buildpackage(dist):
tag = '{}:{}-buildpackage'.format(REGISTRY, dist) tag = "{}:{}-buildpackage".format(REGISTRY, dist)
check_call('docker', 'build', '-t', tag, check_call(
'--build-arg', 'DIST={}'.format(dist), "docker",
'-f', 'Dockerfile.buildpackage', '.') "build",
"-t",
tag,
"--build-arg",
"DIST={}".format(dist),
"-f",
"Dockerfile.buildpackage",
".",
)
def get_cubicweb_images(debian_dists: List[str], python_versions: List[str]): def get_cubicweb_images(debian_dists: List[str], python_versions: List[str]):
images = [] images = []
for dist, python_version in product(debian_dists, python_versions): for dist, python_version in product(debian_dists, python_versions):
python = 'python' if python_version == 'py27' else 'python3' python = "python" if python_version == "py27" else "python3"
check_call( check_call(
'docker', "docker",
'build', "build",
'.', ".",
'-f', "-f",
'Dockerfile.getversion', "Dockerfile.getversion",
'-t', "-t",
'cubicweb:getversion', "cubicweb:getversion",
'--build-arg', "--build-arg",
f'DIST={dist}', f"DIST={dist}",
'--build-arg', "--build-arg",
f"PYTHON={python}" f"PYTHON={python}",
) )
versions = check_output( versions = check_output(
'docker', 'run', '--rm', 'cubicweb:getversion' "docker", "run", "--rm", "cubicweb:getversion"
).splitlines() ).splitlines()
# filter release candidate version # filter release candidate version
valid_versions = [ valid_versions = [version for version in versions if "~rc" not in version]
version for version in versions if '~rc' not in version
]
for version in valid_versions: for version in valid_versions:
image = CubicWebImage( image = CubicWebImage(
python=python_version, python=python_version,
debian=dist, debian=dist,
package_name=f'{python}-cubicweb', package_name=f"{python}-cubicweb",
package_version=version package_version=version,
) )
images.append(image) images.append(image)
return images return images
def build(debian_dists: List[str], images: List[CubicWebImage]=[], rebuild=False):
def build(debian_dists: List[str], images: List[CubicWebImage] = [], rebuild=False):
for dist in debian_dists: for dist in debian_dists:
if rebuild: if rebuild:
# pull base images # pull base images
check_call('docker', 'pull', f'debian:{dist}-slim') check_call("docker", "pull", f"debian:{dist}-slim")
build_buildpackage(dist) build_buildpackage(dist)
for image in images: for image in images:
for image_type in (None, 'onbuild'): for image_type in (None, "onbuild"):
build_image(image, image_type) build_image(image, image_type)
if rebuild and image_type is None: if rebuild and image_type is None:
tag = image.tag tag = image.tag
out = run( out = run(
'docker', 'run', '--rm', '-t', "docker",
'--user', 'root', "run",
'--entrypoint', 'check-docker-updates.sh', "--rm",
tag, 'apt') "-t",
"--user",
"root",
"--entrypoint",
"check-docker-updates.sh",
tag,
"apt",
)
if out.returncode == 1: if out.returncode == 1:
LOG.warning(red( LOG.warning(
'%s debian packages updates are available: %s'), red("%s debian packages updates are available: %s"),
tag, out.stdout) tag,
out.stdout,
)
build_image(image, image_type, no_cache=True) build_image(image, image_type, no_cache=True)
else: else:
assert (out.returncode, out.stdout) == (0, b''), out assert (out.returncode, out.stdout) == (0, b""), out
LOG.info(green('%s debian packages are up-to-date'), tag) LOG.info(green("%s debian packages are up-to-date"), tag)
def push(images: List[CubicWebImage], last_debian_dist: str): def push(images: List[CubicWebImage], last_debian_dist: str):
latest = get_major_tags(images) latest = get_major_tags(images)
for image in images: for image in images:
check_call('docker', 'push', image.tag) check_call("docker", "push", image.tag)
for major, img in latest.items(): for major, img in latest.items():
tag = f"{REGISTRY}:{major}" tag = f"{REGISTRY}:{major}"
for onbuild in [None, 'onbuild']: for onbuild in [None, "onbuild"]:
if onbuild == 'onbuild': if onbuild == "onbuild":
tag = f"{REGISTRY}:{major}-{onbuild}" tag = f"{REGISTRY}:{major}-{onbuild}"
else: else:
tag = f"{REGISTRY}:{major}" tag = f"{REGISTRY}:{major}"
src = img.tag src = img.tag
check_call('docker', 'push', major) check_call("docker", "push", major)
check_call('docker', 'push', img.major_tag) check_call("docker", "push", img.major_tag)
check_call('docker', 'push', f'{REGISTRY}:{last_debian_dist}-buildpackage', f"{REGISTRY}:buildpackage") check_call(
"docker",
"push",
f"{REGISTRY}:{last_debian_dist}-buildpackage",
f"{REGISTRY}:buildpackage",
)
if __name__ == '__main__': if __name__ == "__main__":
import argparse import argparse
logging.basicConfig(format='%(asctime)-15s %(message)s')
logging.basicConfig(format="%(asctime)-15s %(message)s")
LOG.setLevel(logging.INFO) LOG.setLevel(logging.INFO)
parser = argparse.ArgumentParser(sys.argv[0]) parser = argparse.ArgumentParser(sys.argv[0])
parser.add_argument('--registry', action='store', default=REGISTRY) parser.add_argument("--registry", action="store", default=REGISTRY)
parser.add_argument( parser.add_argument(
'--checkrebuild', action='store_true', default=False, "--checkrebuild",
help='Rebuild images in case of outdated debian or python package') action="store_true",
default=False,
help="Rebuild images in case of outdated debian or python package",
)
parser.add_argument( parser.add_argument(
'--push', action='store_true', default=False, "--push", action="store_true", default=False, help="push images"
help='push images') )
args = parser.parse_args() args = parser.parse_args()
REGISTRY = args.registry REGISTRY = args.registry
images = get_cubicweb_images(["stretch", "buster"], ["py27"]) images = get_cubicweb_images(["stretch", "buster"], ["py27"])
images += get_cubicweb_images(["stretch"], ["py35"]) images += get_cubicweb_images(["stretch"], ["py35"])
images += get_cubicweb_images(["buster"], ["py37"]) images += get_cubicweb_images(["buster"], ["py37"])
images.append( images.append(CubicWebImage("py37", "buster", "python3-cubicweb", "dev"))
CubicWebImage("py37", "buster", "python3-cubicweb", "dev")
)
if args.push: if args.push:
push(images=images, last_debian_dist="buster") push(images=images, last_debian_dist="buster")
else: else:
build(debian_dists=["stretch", "buster"], images=images, rebuild=args.checkrebuild) build(
debian_dists=["stretch", "buster"], images=images, rebuild=args.checkrebuild
)
tag_aliases(images=images, last_debian_dist="buster") tag_aliases(images=images, last_debian_dist="buster")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment