Commit ac487a24 authored by Laurent Peuch's avatar Laurent Peuch
Browse files

fix: adding logging in deprecated-warnings.py to try to understand why it...

fix: adding logging in deprecated-warnings.py to try to understand why it fails now without any reason
parent fee4a98b45be
......@@ -3,6 +3,7 @@ import io
import json
import gitlab
import jinja2
import logging
import zipfile
from heptalab import get_all_versions_number_from_pypi
......@@ -15,6 +16,10 @@ from pygments.lexers import get_lexer_by_name
from pygments.formatters.html import HtmlFormatter
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s:%(name)s] %(message)s")
logger = logging.getLogger("dashboard")
def highlight_code(file_content, lineno, full_file=False):
if full_file:
return highlight(file_content, get_lexer_by_name("python", stripnl=False),
......@@ -154,38 +159,55 @@ def render_source_code_file(file_content, path, file_name, lineno):
ALREADY_GENERATED_FILES.add(path)
logger.info("download files_to_url_cache.json...")
download_files_to_url_cache()
files_to_url_cache = json.load(open("files_to_url_cache.json"))
all_artifacts = defaultdict(list)
logger.info("parsing all cubes")
for cube in get_all_cubes():
print(cube.name)
logger.info(f"Working on {cube.name} {cube.web_url}")
branch_default_pipeline = None
logger.info("trying to find latest successful pipeline on branch/default")
for pipeline in cube.pipelines.list(as_list=False):
if pipeline.ref == "branch/default" and pipeline.status == "success":
logger.info(f"found matching pipeline #{pipeline.id} on {pipeline.ref} / {pipeline.status} {pipeline.web_url}")
branch_default_pipeline = pipeline
break
else:
logger.info(f"skip pipeline #{pipeline.id} with ref {pipeline.ref} and status {pipeline.status}")
else:
logger.info(f"no matching pipeline found for cube {cube.name}, skip it")
continue
logger.info("try to extract all artifacts pipeline jobs")
for job in pipeline.jobs.list(all=True):
job = cube.jobs.get(job.id)
logger.info(f"working on job '{job.name}'")
try:
artifacts_archive = zipfile.ZipFile(io.BytesIO(job.artifacts()))
except gitlab.exceptions.GitlabGetError:
logger.info("no artifact on this job, skip")
# 404
continue
logger.info(f"working on all artifacts {artifacts_archive.filelist}...")
for artifact in artifacts_archive.filelist:
if artifact.filename.endswith("deprecated-warnings.json"):
logger.info(f"grab artifact '{artifact.filename}' content...")
content = artifacts_archive.read(artifact.filename).decode()
if not content:
logger.info("no content (artifact is empty), skip it")
continue
content = json.loads(content)
logger.info(f"associate loaded as json artifact content to cube {cube.name}")
all_artifacts[cube].extend(content)
else:
logger.info(f"artifact filename {artifact.filename} doesn't ends with 'deprecated-warnings.json', skip")
result: dict = defaultdict(lambda: {"count": 0, "path": [],
"projects": defaultdict(lambda: {"count": 0,
"dependencies": {},
......@@ -193,13 +215,19 @@ result: dict = defaultdict(lambda: {"count": 0, "path": [],
outdated_projects = []
logger.info("Get all version of numbers of cubicweb from pypi...")
cubicweb_latest_version = get_all_versions_number_from_pypi("cubicweb")[-1]
# yams_latest_version = get_all_versions_number_from_pypi("yams")[-1]
# rql_latest_version = get_all_versions_number_from_pypi("rql")[-1]
# logilab_common_latest_version = get_all_versions_number_from_pypi("logilab-common")[-1]
# logilab_database_latest_version = get_all_versions_number_from_pypi("logilab-database")[-1]
logger.info(f"versions are: {cubicweb_latest_version}")
logger.info("let's parse all artifacts..")
for gitlab_project, warnings in all_artifacts.items():
logger.info(f"working on {gitlab_project.name} {len(warnings)} artifacts files")
for warning in warnings:
if "cubicweb" in warning.get("dependencies", ()):
current_project_cubicweb_version = tuple(map(int, warning["dependencies"]["cubicweb"].split(".")))
......@@ -238,8 +266,10 @@ for gitlab_project, warnings in all_artifacts.items():
if "full_warning" not in result[warning_text]:
result[warning_text]["full_warning"] = warning
result = sorted(result.items(), key=lambda x: -x[1]["count"])
logger.info("sort result")
result = sorted(result.items(), key=lambda x: -x[1]["count"]) # type: ignore
logger.info("render template")
rendered_template = jinja2.Template(open("template/deprecated-warnings.html",
"r").read()).render(warnings=result,
outdated_projects=outdated_projects,
......@@ -250,4 +280,6 @@ rendered_template = jinja2.Template(open("template/deprecated-warnings.html",
render_traceback=render_traceback,
render_source_code_file=render_source_code_file,
pygment_css=HtmlFormatter().get_style_defs())
logger.info("write template to 'public/deprecated-warnings.html'")
open("public/deprecated-warnings.html", "w").write(rendered_template)
logger.info("end")
......@@ -3,6 +3,10 @@
import os
import gitlab
import jinja2
import logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s:%(name)s] %(message)s")
logger = logging.getLogger("dashboard")
gl = gitlab.Gitlab("https://forge.extranet.logilab.fr", oauth_token=os.environ.get("TOKEN"))
......@@ -10,12 +14,15 @@ all_core_projects = [x for x in gl.projects.list(all=True) if "cw-core" in x.tag
context = {"projects": [], "cubes": []}
logger.info("parsing all core projects...")
for project in all_core_projects:
logger.info(f"working on {project.name} {project.web_url}")
# grab the pipelines for default branch that hasn't been triggered from another pipeline
pipeline_for_default = None
for pipeline in project.pipelines.list(as_list=False):
logger.info(f"working on pipeline #{pipeline.id} {pipeline.web_url}")
if pipeline.ref != "branch/default":
continue
......@@ -37,12 +44,15 @@ context["projects"] = sorted(context["projects"], key=lambda x: x["project"]["na
all_cubes = [x for x in gl.projects.list(all=True) if x.namespace["name"] == "cubes"]
logger.info("parsing all cubes...")
for cube in all_cubes:
logger.info(f"working on {cube.name} {cube.web_url}")
# grab the pipelines for default branch that hasn't been triggered from another pipeline
pipeline_for_default = None
for pipeline in cube.pipelines.list(as_list=False, all=True):
logger.info(f"working on pipeline #{pipeline.id} {pipeline.web_url}")
if pipeline.ref != "branch/default":
continue
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment