From 83157234017ac5c7dc7eda0f40c404a9f73f3489 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 27 Apr 2021 17:48:30 +0200 Subject: [PATCH 1/4] feat: add new command regenerate-gitlab-ci --- cube_doctor/doctor_hg.py | 2 + .../transforms/regenerate_gitlab_ci.py | 107 ++++++++++++++++++ tests/data/.gitlab-ci.yml | 20 ++++ tests/data/regenerate_gitlab_ci/in/tox.ini | 69 +++++++++++ .../regenerate_gitlab_ci/out/.gitlab-ci.yml | 25 ++++ tests/data/regenerate_gitlab_ci/out/tox.ini | 69 +++++++++++ tests/test_regenerate_gitlab_ci.py | 20 ++++ 7 files changed, 312 insertions(+) create mode 100644 cube_doctor/transforms/regenerate_gitlab_ci.py create mode 100644 tests/data/.gitlab-ci.yml create mode 100644 tests/data/regenerate_gitlab_ci/in/tox.ini create mode 100644 tests/data/regenerate_gitlab_ci/out/.gitlab-ci.yml create mode 100644 tests/data/regenerate_gitlab_ci/out/tox.ini create mode 100644 tests/test_regenerate_gitlab_ci.py diff --git a/cube_doctor/doctor_hg.py b/cube_doctor/doctor_hg.py index 2928616..a2f3326 100644 --- a/cube_doctor/doctor_hg.py +++ b/cube_doctor/doctor_hg.py @@ -19,6 +19,7 @@ from cube_doctor.transforms.readme_rst import READMERst from cube_doctor.transforms.update_cube_licence_dates import UpdateCubeLicenceDates from cube_doctor.transforms.add_yamllint import AddYamlLint from cube_doctor.transforms.run_script import RunScript +from cube_doctor.transforms.regenerate_gitlab_ci import RegenerateGitlabCI logging.basicConfig(level=logging.INFO) logger = logging.getLogger("cube_doctor") @@ -93,6 +94,7 @@ commands = { "replace-set-attributes": ReplaceSetAttributesWithSetCW().workflow, "update-licence-dates": UpdateCubeLicenceDates().workflow, "auto-upgrade-dependencies": AutoUpgradeDependencies().workflow, + "regenerate-gitlab-ci": RegenerateGitlabCI().workflow, "run-script": RunScript().workflow, # "add-new-version": AddNewVersion().workflow, } diff --git a/cube_doctor/transforms/regenerate_gitlab_ci.py b/cube_doctor/transforms/regenerate_gitlab_ci.py new file mode 100644 index 0000000..8f7b0aa --- /dev/null +++ b/cube_doctor/transforms/regenerate_gitlab_ci.py @@ -0,0 +1,107 @@ +from jinja2 import Environment + +from cube_doctor import Command +from tox.config import parseconfig + +jinja2_env = Environment() + +GITLAB_CI_TEMPLATE = """\ +--- +default: + image: python:3.7 + +include: + - project: "open-source/gitlab-ci-templates" + ref: "branch/default" + file: + - "templates/no-duplicated-ci-pipelines.yml" # use workflow to avoid duplicated pipelines + {%- for file, doc in files_to_include %} + - "templates/{{ file }}"{% if doc %} # {{ doc }}{% endif %} + {%- endfor %} + +stages: + - lint + - tests + - build-debian-package + - upload-deb-to-heptapod + - release + - publish + +""" + + +class RegenerateGitlabCI(Command): + BRANCH_NAME = "topic/default/regenerate_gitlab_ci" + TARGETS = ("cubes",) + commit_message = "ci(gitlab-ci): use templates from a common repository" + base_query = "?project lgg:has_tox true ." + + def pre_check(self, root_files): + if "tox.ini" not in root_files: + return "continue" + + def modify_code(self, cube, repo, root_files, branches, other_args): + jobs = { + "lint": ("flake8", "black", "check-manifest", "mypy", "yamllint"), + "tests": ("py27", "py3"), + } + + tox_ini_path = repo.path / "tox.ini" + tox = parseconfig(["-c", str(tox_ini_path)]) + tox_envs = tox.envconfigs.keys() + + files_to_include = [] + + for category, jobs_names in jobs.items(): + for job_name in jobs_names: + if job_name not in tox_envs: + continue + + files_to_include.append( + ( + f"{category}/{ job_name }.yml", + f"will do the equivalent of 'tox -e { job_name }'", + ) + ) + + if "debian" in root_files: + files_to_include.append( + ( + "build-debian-package.yml", + "will build a .deb and upload it to heptapod files", + ) + ) + files_to_include.append( + ( + "create-release-on-heptapod-including-debian-package.yml", + ( + "this will create a release on heptapod AND uses uploaded .deb by " + "build-debian-package" + ), + ) + ) + else: + files_to_include.append( + ( + "create-release-on-heptapod.yml", + "this will create a release on heptapod", + ) + ) + + if "pypi-publish" in tox_envs: + files_to_include.append( + ( + "upload-to-pypi.yml", + ( + "on a new mercurial tag (expected to be done with release-new), will push " + "a release on pypi" + ), + ) + ) + + new_gitlab_ci = jinja2_env.from_string(GITLAB_CI_TEMPLATE).render( + files_to_include=files_to_include + ) + repo.write_file(".gitlab-ci.yml", new_gitlab_ci) + + yield {"branch_name": self.BRANCH_NAME, "commit_message": self.commit_message} diff --git a/tests/data/.gitlab-ci.yml b/tests/data/.gitlab-ci.yml new file mode 100644 index 0000000..b7ba2d1 --- /dev/null +++ b/tests/data/.gitlab-ci.yml @@ -0,0 +1,20 @@ +default: + image: python + +include: + - project: "open-source/gitlab-templates" + ref: "branch/default" + file: + - "templates/no-duplicate-pipelines.yml" + - "templates/release-on-heptapod.yml" + - "templates/publish-pypi.yml" + +stages: + - tests + - release + - publish + +py3: + stage: tests + command: + - tox -e py3 diff --git a/tests/data/regenerate_gitlab_ci/in/tox.ini b/tests/data/regenerate_gitlab_ci/in/tox.ini new file mode 100644 index 0000000..cbdb77d --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/in/tox.ini @@ -0,0 +1,69 @@ +[tox] +envlist = py3,flake8,check-manifest,black + +[testenv] +deps = + pytest +commands = + {envpython} -m pytest {posargs:test} + +[testenv:flake8] +basepython = python3 +skip_install = true +deps = + flake8 >= 3.6 +commands = flake8 + +[testenv:check-manifest] +skip_install = true +deps = + check-manifest +commands = + {envpython} -m check_manifest {toxinidir} + +[testenv:mypy] +deps = + mypy >= 0.761 +commands = mypy --ignore-missing-imports cubicweb_my_cube + +[testenv:black] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black --check . + +[testenv:black-run] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black . + +[testenv:pypi-publish] +basepython = python3 +skip_install = true +whitelist_externals = rm +deps = + twine +passenv = + TWINE_USERNAME + TWINE_PASSWORD +commands = + rm -rf build dist .egg .egg-info + python3 setup.py sdist bdist_wheel + twine check dist/* + twine upload --skip-existing dist/* + +[testenv:yamllint] +skip_install = true +deps = yamllint +commands = + yamllint . + +[flake8] +basepython = python3 +format = pylint +ignore = W503, E203, E731, E231 +max-line-length = 100 +exclude = cubicweb_my_cube/migration/*,test/data/*,.tox/* diff --git a/tests/data/regenerate_gitlab_ci/out/.gitlab-ci.yml b/tests/data/regenerate_gitlab_ci/out/.gitlab-ci.yml new file mode 100644 index 0000000..6d74d31 --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/out/.gitlab-ci.yml @@ -0,0 +1,25 @@ +--- +default: + image: python:3.7 + +include: + - project: "open-source/gitlab-ci-templates" + ref: "branch/default" + file: + - "templates/no-duplicated-ci-pipelines.yml" # use workflow to avoid duplicated pipelines + - "templates/lint/flake8.yml" # will do the equivalent of 'tox -e flake8' + - "templates/lint/black.yml" # will do the equivalent of 'tox -e black' + - "templates/lint/check-manifest.yml" # will do the equivalent of 'tox -e check-manifest' + - "templates/lint/mypy.yml" # will do the equivalent of 'tox -e mypy' + - "templates/lint/yamllint.yml" # will do the equivalent of 'tox -e yamllint' + - "templates/tests/py3.yml" # will do the equivalent of 'tox -e py3' + - "templates/create-release-on-heptapod.yml" # this will create a release on heptapod + - "templates/upload-to-pypi.yml" # on a new mercurial tag (expected to be done with release-new), will push a release on pypi + +stages: + - lint + - tests + - build-debian-package + - upload-deb-to-heptapod + - release + - publish diff --git a/tests/data/regenerate_gitlab_ci/out/tox.ini b/tests/data/regenerate_gitlab_ci/out/tox.ini new file mode 100644 index 0000000..cbdb77d --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/out/tox.ini @@ -0,0 +1,69 @@ +[tox] +envlist = py3,flake8,check-manifest,black + +[testenv] +deps = + pytest +commands = + {envpython} -m pytest {posargs:test} + +[testenv:flake8] +basepython = python3 +skip_install = true +deps = + flake8 >= 3.6 +commands = flake8 + +[testenv:check-manifest] +skip_install = true +deps = + check-manifest +commands = + {envpython} -m check_manifest {toxinidir} + +[testenv:mypy] +deps = + mypy >= 0.761 +commands = mypy --ignore-missing-imports cubicweb_my_cube + +[testenv:black] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black --check . + +[testenv:black-run] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black . + +[testenv:pypi-publish] +basepython = python3 +skip_install = true +whitelist_externals = rm +deps = + twine +passenv = + TWINE_USERNAME + TWINE_PASSWORD +commands = + rm -rf build dist .egg .egg-info + python3 setup.py sdist bdist_wheel + twine check dist/* + twine upload --skip-existing dist/* + +[testenv:yamllint] +skip_install = true +deps = yamllint +commands = + yamllint . + +[flake8] +basepython = python3 +format = pylint +ignore = W503, E203, E731, E231 +max-line-length = 100 +exclude = cubicweb_my_cube/migration/*,test/data/*,.tox/* diff --git a/tests/test_regenerate_gitlab_ci.py b/tests/test_regenerate_gitlab_ci.py new file mode 100644 index 0000000..067850c --- /dev/null +++ b/tests/test_regenerate_gitlab_ci.py @@ -0,0 +1,20 @@ +import unittest + +from cube_doctor.transforms.regenerate_gitlab_ci import RegenerateGitlabCI + +from . import BaseCommandTC, TEST_DATA + + +class TransformRegenerateGitlabCITC(BaseCommandTC): + def test_regenerate_gitlab_ci(self): + data_input = TEST_DATA / "regenerate_gitlab_ci" / "in" + data_output = TEST_DATA / "regenerate_gitlab_ci" / "out" + self.base_execution(RegenerateGitlabCI(), data_input, data_output) + + def test_regenerate_gitlab_ci_idempotent(self): + data_output = TEST_DATA / "regenerate_gitlab_ci" / "out" + self.base_execution(RegenerateGitlabCI(), data_output, data_output) + + +if __name__ == "__main__": + unittest.main() -- GitLab From a00de5ec704e8a59e3224220dd1712050ee0e5d2 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 8 Jul 2021 03:25:30 +0200 Subject: [PATCH 2/4] feat(regenerate-gitlab-ci.yml): let's support safety too --- cube_doctor/transforms/regenerate_gitlab_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cube_doctor/transforms/regenerate_gitlab_ci.py b/cube_doctor/transforms/regenerate_gitlab_ci.py index 8f7b0aa..42a8273 100644 --- a/cube_doctor/transforms/regenerate_gitlab_ci.py +++ b/cube_doctor/transforms/regenerate_gitlab_ci.py @@ -42,7 +42,7 @@ class RegenerateGitlabCI(Command): def modify_code(self, cube, repo, root_files, branches, other_args): jobs = { - "lint": ("flake8", "black", "check-manifest", "mypy", "yamllint"), + "lint": ("flake8", "black", "check-manifest", "mypy", "yamllint", "safety"), "tests": ("py27", "py3"), } -- GitLab From 13841214270ae19c3bde569b0848473f84278481 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 29 Apr 2021 03:27:29 +0200 Subject: [PATCH 3/4] feat(regenerate-gitlab-ci): add support for a .gitlab-ci-extended.yml --- .../transforms/regenerate_gitlab_ci.py | 37 +++++++++- .../in_extended/.gitlab-ci.yml | 26 +++++++ .../regenerate_gitlab_ci/in_extended/tox.ini | 69 +++++++++++++++++++ .../out/.gitlab-ci-extended.yml | 10 +++ .../regenerate_gitlab_ci/out/.gitlab-ci.yml | 3 + .../out_extended/.gitlab-ci-extended.yml | 10 +++ .../out_extended/.gitlab-ci.yml | 26 +++++++ .../regenerate_gitlab_ci/out_extended/tox.ini | 69 +++++++++++++++++++ tests/test_regenerate_gitlab_ci.py | 9 +++ 9 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 tests/data/regenerate_gitlab_ci/in_extended/.gitlab-ci.yml create mode 100644 tests/data/regenerate_gitlab_ci/in_extended/tox.ini create mode 100644 tests/data/regenerate_gitlab_ci/out/.gitlab-ci-extended.yml create mode 100644 tests/data/regenerate_gitlab_ci/out_extended/.gitlab-ci-extended.yml create mode 100644 tests/data/regenerate_gitlab_ci/out_extended/.gitlab-ci.yml create mode 100644 tests/data/regenerate_gitlab_ci/out_extended/tox.ini diff --git a/cube_doctor/transforms/regenerate_gitlab_ci.py b/cube_doctor/transforms/regenerate_gitlab_ci.py index 42a8273..497d7d7 100644 --- a/cube_doctor/transforms/regenerate_gitlab_ci.py +++ b/cube_doctor/transforms/regenerate_gitlab_ci.py @@ -1,3 +1,5 @@ +import yaml + from jinja2 import Environment from cube_doctor import Command @@ -11,6 +13,13 @@ default: image: python:3.7 include: + {%- if not uses_gitlab_ci_extended %} + # uncomment and uses to customize/extend the configuration here if needed + # (it needs to be at the same level than "- project") + # - ".gitlab-ci-extended.yml" + {% else %} + - ".gitlab-ci-extended.yml" + {% endif -%} - project: "open-source/gitlab-ci-templates" ref: "branch/default" file: @@ -29,6 +38,19 @@ stages: """ +GITLAB_CI_EXTENDED_TEMPLATE = """\ +--- +# follow this doc to extend the already existing jobs: +# https://docs.gitlab.com/ee/ci/yaml/includes.html#overriding-external-template-values + +# as a reminder the existing jobs are located here: +# https://forge.extranet.logilab.fr/open-source/gitlab-templates/ + +# or simply add more jobs here if you need them (use the already existing stages) + +# /!\\ remember to uncomment the include in the .gitlab-ci.yml /!\\ +""" + class RegenerateGitlabCI(Command): BRANCH_NAME = "topic/default/regenerate_gitlab_ci" @@ -99,9 +121,22 @@ class RegenerateGitlabCI(Command): ) ) + if ".gitlab-ci.yml" in root_files: + loaded_gitlab_ci = yaml.safe_load(repo.read_file(".gitlab-ci.yml")) + uses_gitlab_ci_extended = ".gitlab-ci-extended.yml" in loaded_gitlab_ci.get( + "include", [] + ) + else: + uses_gitlab_ci_extended = False + new_gitlab_ci = jinja2_env.from_string(GITLAB_CI_TEMPLATE).render( - files_to_include=files_to_include + files_to_include=files_to_include, + uses_gitlab_ci_extended=uses_gitlab_ci_extended, ) repo.write_file(".gitlab-ci.yml", new_gitlab_ci) + if ".gitlab-ci-extended.yml" not in root_files: + repo.write_file(".gitlab-ci-extended.yml", GITLAB_CI_EXTENDED_TEMPLATE) + repo.run_command("hg add .gitlab-ci-extended.yml") + yield {"branch_name": self.BRANCH_NAME, "commit_message": self.commit_message} diff --git a/tests/data/regenerate_gitlab_ci/in_extended/.gitlab-ci.yml b/tests/data/regenerate_gitlab_ci/in_extended/.gitlab-ci.yml new file mode 100644 index 0000000..c5df843 --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/in_extended/.gitlab-ci.yml @@ -0,0 +1,26 @@ +--- +default: + image: python:3.7 + +include: + - ".gitlab-ci-extended.yml" + - project: "open-source/gitlab-ci-templates" + ref: "branch/default" + file: + - "templates/no-duplicated-ci-pipelines.yml" # use workflow to avoid duplicated pipelines + - "templates/lint/flake8.yml" # will do the equivalent of 'tox -e flake8' + - "templates/lint/black.yml" # will do the equivalent of 'tox -e black' + - "templates/lint/check-manifest.yml" # will do the equivalent of 'tox -e check-manifest' + - "templates/lint/mypy.yml" # will do the equivalent of 'tox -e mypy' + - "templates/lint/yamllint.yml" # will do the equivalent of 'tox -e yamllint' + - "templates/tests/py3.yml" # will do the equivalent of 'tox -e py3' + - "templates/create-release-on-heptapod.yml" # this will create a release on heptapod + - "templates/upload-to-pypi.yml" # on a new mercurial tag (expected to be done with release-new), will push a release on pypi + +stages: + - lint + - tests + - build-debian-package + - upload-deb-to-heptapod + - release + - publish diff --git a/tests/data/regenerate_gitlab_ci/in_extended/tox.ini b/tests/data/regenerate_gitlab_ci/in_extended/tox.ini new file mode 100644 index 0000000..cbdb77d --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/in_extended/tox.ini @@ -0,0 +1,69 @@ +[tox] +envlist = py3,flake8,check-manifest,black + +[testenv] +deps = + pytest +commands = + {envpython} -m pytest {posargs:test} + +[testenv:flake8] +basepython = python3 +skip_install = true +deps = + flake8 >= 3.6 +commands = flake8 + +[testenv:check-manifest] +skip_install = true +deps = + check-manifest +commands = + {envpython} -m check_manifest {toxinidir} + +[testenv:mypy] +deps = + mypy >= 0.761 +commands = mypy --ignore-missing-imports cubicweb_my_cube + +[testenv:black] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black --check . + +[testenv:black-run] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black . + +[testenv:pypi-publish] +basepython = python3 +skip_install = true +whitelist_externals = rm +deps = + twine +passenv = + TWINE_USERNAME + TWINE_PASSWORD +commands = + rm -rf build dist .egg .egg-info + python3 setup.py sdist bdist_wheel + twine check dist/* + twine upload --skip-existing dist/* + +[testenv:yamllint] +skip_install = true +deps = yamllint +commands = + yamllint . + +[flake8] +basepython = python3 +format = pylint +ignore = W503, E203, E731, E231 +max-line-length = 100 +exclude = cubicweb_my_cube/migration/*,test/data/*,.tox/* diff --git a/tests/data/regenerate_gitlab_ci/out/.gitlab-ci-extended.yml b/tests/data/regenerate_gitlab_ci/out/.gitlab-ci-extended.yml new file mode 100644 index 0000000..f22c19c --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/out/.gitlab-ci-extended.yml @@ -0,0 +1,10 @@ +--- +# follow this doc to extend the already existing jobs: +# https://docs.gitlab.com/ee/ci/yaml/includes.html#overriding-external-template-values + +# as a reminder the existing jobs are located here: +# https://forge.extranet.logilab.fr/open-source/gitlab-templates/ + +# or simply add more jobs here if you need them (use the already existing stages) + +# /!\ remember to uncomment the include in the .gitlab-ci.yml /!\ diff --git a/tests/data/regenerate_gitlab_ci/out/.gitlab-ci.yml b/tests/data/regenerate_gitlab_ci/out/.gitlab-ci.yml index 6d74d31..05166d3 100644 --- a/tests/data/regenerate_gitlab_ci/out/.gitlab-ci.yml +++ b/tests/data/regenerate_gitlab_ci/out/.gitlab-ci.yml @@ -3,6 +3,9 @@ default: image: python:3.7 include: + # uncomment and uses to customize/extend the configuration here if needed + # (it needs to be at the same level than "- project") + # - ".gitlab-ci-extended.yml" - project: "open-source/gitlab-ci-templates" ref: "branch/default" file: diff --git a/tests/data/regenerate_gitlab_ci/out_extended/.gitlab-ci-extended.yml b/tests/data/regenerate_gitlab_ci/out_extended/.gitlab-ci-extended.yml new file mode 100644 index 0000000..f22c19c --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/out_extended/.gitlab-ci-extended.yml @@ -0,0 +1,10 @@ +--- +# follow this doc to extend the already existing jobs: +# https://docs.gitlab.com/ee/ci/yaml/includes.html#overriding-external-template-values + +# as a reminder the existing jobs are located here: +# https://forge.extranet.logilab.fr/open-source/gitlab-templates/ + +# or simply add more jobs here if you need them (use the already existing stages) + +# /!\ remember to uncomment the include in the .gitlab-ci.yml /!\ diff --git a/tests/data/regenerate_gitlab_ci/out_extended/.gitlab-ci.yml b/tests/data/regenerate_gitlab_ci/out_extended/.gitlab-ci.yml new file mode 100644 index 0000000..c5df843 --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/out_extended/.gitlab-ci.yml @@ -0,0 +1,26 @@ +--- +default: + image: python:3.7 + +include: + - ".gitlab-ci-extended.yml" + - project: "open-source/gitlab-ci-templates" + ref: "branch/default" + file: + - "templates/no-duplicated-ci-pipelines.yml" # use workflow to avoid duplicated pipelines + - "templates/lint/flake8.yml" # will do the equivalent of 'tox -e flake8' + - "templates/lint/black.yml" # will do the equivalent of 'tox -e black' + - "templates/lint/check-manifest.yml" # will do the equivalent of 'tox -e check-manifest' + - "templates/lint/mypy.yml" # will do the equivalent of 'tox -e mypy' + - "templates/lint/yamllint.yml" # will do the equivalent of 'tox -e yamllint' + - "templates/tests/py3.yml" # will do the equivalent of 'tox -e py3' + - "templates/create-release-on-heptapod.yml" # this will create a release on heptapod + - "templates/upload-to-pypi.yml" # on a new mercurial tag (expected to be done with release-new), will push a release on pypi + +stages: + - lint + - tests + - build-debian-package + - upload-deb-to-heptapod + - release + - publish diff --git a/tests/data/regenerate_gitlab_ci/out_extended/tox.ini b/tests/data/regenerate_gitlab_ci/out_extended/tox.ini new file mode 100644 index 0000000..cbdb77d --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/out_extended/tox.ini @@ -0,0 +1,69 @@ +[tox] +envlist = py3,flake8,check-manifest,black + +[testenv] +deps = + pytest +commands = + {envpython} -m pytest {posargs:test} + +[testenv:flake8] +basepython = python3 +skip_install = true +deps = + flake8 >= 3.6 +commands = flake8 + +[testenv:check-manifest] +skip_install = true +deps = + check-manifest +commands = + {envpython} -m check_manifest {toxinidir} + +[testenv:mypy] +deps = + mypy >= 0.761 +commands = mypy --ignore-missing-imports cubicweb_my_cube + +[testenv:black] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black --check . + +[testenv:black-run] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black . + +[testenv:pypi-publish] +basepython = python3 +skip_install = true +whitelist_externals = rm +deps = + twine +passenv = + TWINE_USERNAME + TWINE_PASSWORD +commands = + rm -rf build dist .egg .egg-info + python3 setup.py sdist bdist_wheel + twine check dist/* + twine upload --skip-existing dist/* + +[testenv:yamllint] +skip_install = true +deps = yamllint +commands = + yamllint . + +[flake8] +basepython = python3 +format = pylint +ignore = W503, E203, E731, E231 +max-line-length = 100 +exclude = cubicweb_my_cube/migration/*,test/data/*,.tox/* diff --git a/tests/test_regenerate_gitlab_ci.py b/tests/test_regenerate_gitlab_ci.py index 067850c..e64a960 100644 --- a/tests/test_regenerate_gitlab_ci.py +++ b/tests/test_regenerate_gitlab_ci.py @@ -15,6 +15,15 @@ class TransformRegenerateGitlabCITC(BaseCommandTC): data_output = TEST_DATA / "regenerate_gitlab_ci" / "out" self.base_execution(RegenerateGitlabCI(), data_output, data_output) + def test_regenerate_gitlab_ci_extended(self): + data_input = TEST_DATA / "regenerate_gitlab_ci" / "in_extended" + data_output = TEST_DATA / "regenerate_gitlab_ci" / "out_extended" + self.base_execution(RegenerateGitlabCI(), data_input, data_output) + + def test_regenerate_gitlab_ci_extended_idempotent(self): + data_output = TEST_DATA / "regenerate_gitlab_ci" / "out_extended" + self.base_execution(RegenerateGitlabCI(), data_output, data_output) + if __name__ == "__main__": unittest.main() -- GitLab From 7196c4f9650fe651a46842cfc442ea44643b35b3 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 8 Jul 2021 04:42:00 +0200 Subject: [PATCH 4/4] feat(regenerate_gitlab_ci): handle MANIFEST.in --- .../transforms/regenerate_gitlab_ci.py | 14 ++++ .../in_manifest_in/MANIFEST.in | 0 .../in_manifest_in/tox.ini | 69 +++++++++++++++++++ .../out_manifest_in/.gitlab-ci-extended.yml | 10 +++ .../out_manifest_in/.gitlab-ci.yml | 28 ++++++++ .../out_manifest_in/MANIFEST.in | 3 + .../out_manifest_in/tox.ini | 69 +++++++++++++++++++ tests/test_regenerate_gitlab_ci.py | 9 +++ 8 files changed, 202 insertions(+) create mode 100644 tests/data/regenerate_gitlab_ci/in_manifest_in/MANIFEST.in create mode 100644 tests/data/regenerate_gitlab_ci/in_manifest_in/tox.ini create mode 100644 tests/data/regenerate_gitlab_ci/out_manifest_in/.gitlab-ci-extended.yml create mode 100644 tests/data/regenerate_gitlab_ci/out_manifest_in/.gitlab-ci.yml create mode 100644 tests/data/regenerate_gitlab_ci/out_manifest_in/MANIFEST.in create mode 100644 tests/data/regenerate_gitlab_ci/out_manifest_in/tox.ini diff --git a/cube_doctor/transforms/regenerate_gitlab_ci.py b/cube_doctor/transforms/regenerate_gitlab_ci.py index 497d7d7..6750950 100644 --- a/cube_doctor/transforms/regenerate_gitlab_ci.py +++ b/cube_doctor/transforms/regenerate_gitlab_ci.py @@ -135,8 +135,22 @@ class RegenerateGitlabCI(Command): ) repo.write_file(".gitlab-ci.yml", new_gitlab_ci) + if "MANIFEST.in" in root_files and ".gitlab-ci.yml" not in repo.read_file( + "MANIFEST.in" + ): + manifest_content = repo.read_file("MANIFEST.in") + manifest_content = manifest_content.rstrip() + "\nexclude .gitlab-ci.yml\n" + repo.write_file("MANIFEST.in", manifest_content) + if ".gitlab-ci-extended.yml" not in root_files: repo.write_file(".gitlab-ci-extended.yml", GITLAB_CI_EXTENDED_TEMPLATE) repo.run_command("hg add .gitlab-ci-extended.yml") + if "MANIFEST.in" in root_files: + manifest_content = repo.read_file("MANIFEST.in") + manifest_content = ( + manifest_content.rstrip() + "\nexclude .gitlab-ci-extended.yml\n" + ) + repo.write_file("MANIFEST.in", manifest_content) + yield {"branch_name": self.BRANCH_NAME, "commit_message": self.commit_message} diff --git a/tests/data/regenerate_gitlab_ci/in_manifest_in/MANIFEST.in b/tests/data/regenerate_gitlab_ci/in_manifest_in/MANIFEST.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/regenerate_gitlab_ci/in_manifest_in/tox.ini b/tests/data/regenerate_gitlab_ci/in_manifest_in/tox.ini new file mode 100644 index 0000000..cbdb77d --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/in_manifest_in/tox.ini @@ -0,0 +1,69 @@ +[tox] +envlist = py3,flake8,check-manifest,black + +[testenv] +deps = + pytest +commands = + {envpython} -m pytest {posargs:test} + +[testenv:flake8] +basepython = python3 +skip_install = true +deps = + flake8 >= 3.6 +commands = flake8 + +[testenv:check-manifest] +skip_install = true +deps = + check-manifest +commands = + {envpython} -m check_manifest {toxinidir} + +[testenv:mypy] +deps = + mypy >= 0.761 +commands = mypy --ignore-missing-imports cubicweb_my_cube + +[testenv:black] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black --check . + +[testenv:black-run] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black . + +[testenv:pypi-publish] +basepython = python3 +skip_install = true +whitelist_externals = rm +deps = + twine +passenv = + TWINE_USERNAME + TWINE_PASSWORD +commands = + rm -rf build dist .egg .egg-info + python3 setup.py sdist bdist_wheel + twine check dist/* + twine upload --skip-existing dist/* + +[testenv:yamllint] +skip_install = true +deps = yamllint +commands = + yamllint . + +[flake8] +basepython = python3 +format = pylint +ignore = W503, E203, E731, E231 +max-line-length = 100 +exclude = cubicweb_my_cube/migration/*,test/data/*,.tox/* diff --git a/tests/data/regenerate_gitlab_ci/out_manifest_in/.gitlab-ci-extended.yml b/tests/data/regenerate_gitlab_ci/out_manifest_in/.gitlab-ci-extended.yml new file mode 100644 index 0000000..f22c19c --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/out_manifest_in/.gitlab-ci-extended.yml @@ -0,0 +1,10 @@ +--- +# follow this doc to extend the already existing jobs: +# https://docs.gitlab.com/ee/ci/yaml/includes.html#overriding-external-template-values + +# as a reminder the existing jobs are located here: +# https://forge.extranet.logilab.fr/open-source/gitlab-templates/ + +# or simply add more jobs here if you need them (use the already existing stages) + +# /!\ remember to uncomment the include in the .gitlab-ci.yml /!\ diff --git a/tests/data/regenerate_gitlab_ci/out_manifest_in/.gitlab-ci.yml b/tests/data/regenerate_gitlab_ci/out_manifest_in/.gitlab-ci.yml new file mode 100644 index 0000000..05166d3 --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/out_manifest_in/.gitlab-ci.yml @@ -0,0 +1,28 @@ +--- +default: + image: python:3.7 + +include: + # uncomment and uses to customize/extend the configuration here if needed + # (it needs to be at the same level than "- project") + # - ".gitlab-ci-extended.yml" + - project: "open-source/gitlab-ci-templates" + ref: "branch/default" + file: + - "templates/no-duplicated-ci-pipelines.yml" # use workflow to avoid duplicated pipelines + - "templates/lint/flake8.yml" # will do the equivalent of 'tox -e flake8' + - "templates/lint/black.yml" # will do the equivalent of 'tox -e black' + - "templates/lint/check-manifest.yml" # will do the equivalent of 'tox -e check-manifest' + - "templates/lint/mypy.yml" # will do the equivalent of 'tox -e mypy' + - "templates/lint/yamllint.yml" # will do the equivalent of 'tox -e yamllint' + - "templates/tests/py3.yml" # will do the equivalent of 'tox -e py3' + - "templates/create-release-on-heptapod.yml" # this will create a release on heptapod + - "templates/upload-to-pypi.yml" # on a new mercurial tag (expected to be done with release-new), will push a release on pypi + +stages: + - lint + - tests + - build-debian-package + - upload-deb-to-heptapod + - release + - publish diff --git a/tests/data/regenerate_gitlab_ci/out_manifest_in/MANIFEST.in b/tests/data/regenerate_gitlab_ci/out_manifest_in/MANIFEST.in new file mode 100644 index 0000000..00b4233 --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/out_manifest_in/MANIFEST.in @@ -0,0 +1,3 @@ + +exclude .gitlab-ci.yml +exclude .gitlab-ci-extended.yml diff --git a/tests/data/regenerate_gitlab_ci/out_manifest_in/tox.ini b/tests/data/regenerate_gitlab_ci/out_manifest_in/tox.ini new file mode 100644 index 0000000..cbdb77d --- /dev/null +++ b/tests/data/regenerate_gitlab_ci/out_manifest_in/tox.ini @@ -0,0 +1,69 @@ +[tox] +envlist = py3,flake8,check-manifest,black + +[testenv] +deps = + pytest +commands = + {envpython} -m pytest {posargs:test} + +[testenv:flake8] +basepython = python3 +skip_install = true +deps = + flake8 >= 3.6 +commands = flake8 + +[testenv:check-manifest] +skip_install = true +deps = + check-manifest +commands = + {envpython} -m check_manifest {toxinidir} + +[testenv:mypy] +deps = + mypy >= 0.761 +commands = mypy --ignore-missing-imports cubicweb_my_cube + +[testenv:black] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black --check . + +[testenv:black-run] +basepython = python3 +skip_install = true +deps = + black >= 19.10b0 +commands = black . + +[testenv:pypi-publish] +basepython = python3 +skip_install = true +whitelist_externals = rm +deps = + twine +passenv = + TWINE_USERNAME + TWINE_PASSWORD +commands = + rm -rf build dist .egg .egg-info + python3 setup.py sdist bdist_wheel + twine check dist/* + twine upload --skip-existing dist/* + +[testenv:yamllint] +skip_install = true +deps = yamllint +commands = + yamllint . + +[flake8] +basepython = python3 +format = pylint +ignore = W503, E203, E731, E231 +max-line-length = 100 +exclude = cubicweb_my_cube/migration/*,test/data/*,.tox/* diff --git a/tests/test_regenerate_gitlab_ci.py b/tests/test_regenerate_gitlab_ci.py index e64a960..9958d2d 100644 --- a/tests/test_regenerate_gitlab_ci.py +++ b/tests/test_regenerate_gitlab_ci.py @@ -15,6 +15,15 @@ class TransformRegenerateGitlabCITC(BaseCommandTC): data_output = TEST_DATA / "regenerate_gitlab_ci" / "out" self.base_execution(RegenerateGitlabCI(), data_output, data_output) + def test_regenerate_gitlab_ci_manifest_in(self): + data_input = TEST_DATA / "regenerate_gitlab_ci" / "in_manifest_in" + data_output = TEST_DATA / "regenerate_gitlab_ci" / "out_manifest_in" + self.base_execution(RegenerateGitlabCI(), data_input, data_output) + + def test_regenerate_gitlab_ci_manifest_in_idempotent(self): + data_output = TEST_DATA / "regenerate_gitlab_ci" / "out_manifest_in" + self.base_execution(RegenerateGitlabCI(), data_output, data_output) + def test_regenerate_gitlab_ci_extended(self): data_input = TEST_DATA / "regenerate_gitlab_ci" / "in_extended" data_output = TEST_DATA / "regenerate_gitlab_ci" / "out_extended" -- GitLab