fix: adding logging in deprecated-warnings.py to try to understand why it...

fix: adding logging in deprecated-warnings.py to try to understand why it fails now without any reason

deprecated-warnings.py

@@ -3,6 +3,7 @@ import io
 import json
 import gitlab
 import jinja2
+import logging
 import zipfile
 
 from heptalab import get_all_versions_number_from_pypi
@@ -15,6 +16,10 @@ from pygments.lexers import get_lexer_by_name
 from pygments.formatters.html import HtmlFormatter
 
 
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s:%(name)s] %(message)s")
+logger = logging.getLogger("dashboard")
+
+
 def highlight_code(file_content, lineno, full_file=False):
     if full_file:
         return highlight(file_content, get_lexer_by_name("python", stripnl=False),
@@ -154,38 +159,55 @@ def render_source_code_file(file_content, path, file_name, lineno):
     ALREADY_GENERATED_FILES.add(path)
 
 
+logger.info("download files_to_url_cache.json...")
 download_files_to_url_cache()
 files_to_url_cache = json.load(open("files_to_url_cache.json"))
 
 all_artifacts = defaultdict(list)
 
+logger.info("parsing all cubes")
 for cube in get_all_cubes():
-    print(cube.name)
+    logger.info(f"Working on {cube.name} {cube.web_url}")
     branch_default_pipeline = None
+    logger.info("trying to find latest successful pipeline on branch/default")
     for pipeline in cube.pipelines.list(as_list=False):
         if pipeline.ref == "branch/default" and pipeline.status == "success":
+            logger.info(f"found matching pipeline #{pipeline.id} on {pipeline.ref} / {pipeline.status} {pipeline.web_url}")
             branch_default_pipeline = pipeline
             break
+        else:
+            logger.info(f"skip pipeline #{pipeline.id} with ref {pipeline.ref} and status {pipeline.status}")
     else:
+        logger.info(f"no matching pipeline found for cube {cube.name}, skip it")
         continue
 
+    logger.info("try to extract all artifacts pipeline jobs")
     for job in pipeline.jobs.list(all=True):
         job = cube.jobs.get(job.id)
+        logger.info(f"working on job '{job.name}'")
         try:
             artifacts_archive = zipfile.ZipFile(io.BytesIO(job.artifacts()))
         except gitlab.exceptions.GitlabGetError:
+            logger.info("no artifact on this job, skip")
             # 404
             continue
 
+        logger.info(f"working on all artifacts {artifacts_archive.filelist}...")
         for artifact in artifacts_archive.filelist:
             if artifact.filename.endswith("deprecated-warnings.json"):
+                logger.info(f"grab artifact '{artifact.filename}' content...")
                 content = artifacts_archive.read(artifact.filename).decode()
                 if not content:
+                    logger.info("no content (artifact is empty), skip it")
                     continue
 
                 content = json.loads(content)
+                logger.info(f"associate loaded as json artifact content to cube {cube.name}")
                 all_artifacts[cube].extend(content)
 
+            else:
+                logger.info(f"artifact filename {artifact.filename} doesn't ends with 'deprecated-warnings.json', skip")
+
 result: dict = defaultdict(lambda: {"count": 0, "path": [],
                                     "projects": defaultdict(lambda: {"count": 0,
                                                                      "dependencies": {},
@@ -193,13 +215,19 @@ result: dict = defaultdict(lambda: {"count": 0, "path": [],
 
 outdated_projects = []
 
+logger.info("Get all version of numbers of cubicweb from pypi...")
 cubicweb_latest_version = get_all_versions_number_from_pypi("cubicweb")[-1]
 # yams_latest_version = get_all_versions_number_from_pypi("yams")[-1]
 # rql_latest_version = get_all_versions_number_from_pypi("rql")[-1]
 # logilab_common_latest_version = get_all_versions_number_from_pypi("logilab-common")[-1]
 # logilab_database_latest_version = get_all_versions_number_from_pypi("logilab-database")[-1]
 
+logger.info(f"versions are: {cubicweb_latest_version}")
+
+logger.info("let's parse all artifacts..")
+
 for gitlab_project, warnings in all_artifacts.items():
+    logger.info(f"working on {gitlab_project.name} {len(warnings)} artifacts files")
     for warning in warnings:
         if "cubicweb" in warning.get("dependencies", ()):
             current_project_cubicweb_version = tuple(map(int, warning["dependencies"]["cubicweb"].split(".")))
@@ -238,8 +266,10 @@ for gitlab_project, warnings in all_artifacts.items():
             if "full_warning" not in result[warning_text]:
                 result[warning_text]["full_warning"] = warning
 
-result = sorted(result.items(), key=lambda x: -x[1]["count"])
+logger.info("sort result")
+result = sorted(result.items(), key=lambda x: -x[1]["count"])  # type: ignore
 
+logger.info("render template")
 rendered_template = jinja2.Template(open("template/deprecated-warnings.html",
                                          "r").read()).render(warnings=result,
                                                              outdated_projects=outdated_projects,
@@ -250,4 +280,6 @@ rendered_template = jinja2.Template(open("template/deprecated-warnings.html",
                                                              render_traceback=render_traceback,
                                                              render_source_code_file=render_source_code_file,
                                                              pygment_css=HtmlFormatter().get_style_defs())
+logger.info("write template to 'public/deprecated-warnings.html'")
 open("public/deprecated-warnings.html", "w").write(rendered_template)
+logger.info("end")

pipelines_status.py

@@ -3,6 +3,10 @@
 import os
 import gitlab
 import jinja2
+import logging
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s:%(name)s] %(message)s")
+logger = logging.getLogger("dashboard")
 
 gl = gitlab.Gitlab("https://forge.extranet.logilab.fr", oauth_token=os.environ.get("TOKEN"))
 
@@ -10,12 +14,15 @@ all_core_projects = [x for x in gl.projects.list(all=True) if "cw-core" in x.tag
 
 context = {"projects": [], "cubes": []}
 
+logger.info("parsing all core projects...")
 for project in all_core_projects:
+    logger.info(f"working on {project.name} {project.web_url}")
 
     # grab the pipelines for default branch that hasn't been triggered from another pipeline
     pipeline_for_default = None
 
     for pipeline in project.pipelines.list(as_list=False):
+        logger.info(f"working on pipeline #{pipeline.id} {pipeline.web_url}")
         if pipeline.ref != "branch/default":
             continue
 
@@ -37,12 +44,15 @@ context["projects"] = sorted(context["projects"], key=lambda x: x["project"]["na
 
 all_cubes = [x for x in gl.projects.list(all=True) if x.namespace["name"] == "cubes"]
 
+logger.info("parsing all cubes...")
 for cube in all_cubes:
+    logger.info(f"working on {cube.name} {cube.web_url}")
 
     # grab the pipelines for default branch that hasn't been triggered from another pipeline
     pipeline_for_default = None
 
     for pipeline in cube.pipelines.list(as_list=False, all=True):
+        logger.info(f"working on pipeline #{pipeline.id} {pipeline.web_url}")
         if pipeline.ref != "branch/default":
             continue