Commit ac487a24 authored by Laurent Peuch's avatar Laurent Peuch
Browse files

fix: adding logging in to try to understand why it...

fix: adding logging in to try to understand why it fails now without any reason
parent fee4a98b45be
......@@ -3,6 +3,7 @@ import io
import json
import gitlab
import jinja2
import logging
import zipfile
from heptalab import get_all_versions_number_from_pypi
......@@ -15,6 +16,10 @@ from pygments.lexers import get_lexer_by_name
from pygments.formatters.html import HtmlFormatter
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s:%(name)s] %(message)s")
logger = logging.getLogger("dashboard")
def highlight_code(file_content, lineno, full_file=False):
if full_file:
return highlight(file_content, get_lexer_by_name("python", stripnl=False),
......@@ -154,38 +159,55 @@ def render_source_code_file(file_content, path, file_name, lineno):
ALREADY_GENERATED_FILES.add(path)"download files_to_url_cache.json...")
files_to_url_cache = json.load(open("files_to_url_cache.json"))
all_artifacts = defaultdict(list)"parsing all cubes")
for cube in get_all_cubes():
print("Working on {} {cube.web_url}")
branch_default_pipeline = None"trying to find latest successful pipeline on branch/default")
for pipeline in cube.pipelines.list(as_list=False):
if pipeline.ref == "branch/default" and pipeline.status == "success":"found matching pipeline #{} on {pipeline.ref} / {pipeline.status} {pipeline.web_url}")
branch_default_pipeline = pipeline
else:"skip pipeline #{} with ref {pipeline.ref} and status {pipeline.status}")
else:"no matching pipeline found for cube {}, skip it")
continue"try to extract all artifacts pipeline jobs")
for job in
job ="working on job '{}'")
artifacts_archive = zipfile.ZipFile(io.BytesIO(job.artifacts()))
except gitlab.exceptions.GitlabGetError:"no artifact on this job, skip")
# 404
continue"working on all artifacts {artifacts_archive.filelist}...")
for artifact in artifacts_archive.filelist:
if artifact.filename.endswith("deprecated-warnings.json"):"grab artifact '{artifact.filename}' content...")
content =
if not content:"no content (artifact is empty), skip it")
content = json.loads(content)"associate loaded as json artifact content to cube {}")
else:"artifact filename {artifact.filename} doesn't ends with 'deprecated-warnings.json', skip")
result: dict = defaultdict(lambda: {"count": 0, "path": [],
"projects": defaultdict(lambda: {"count": 0,
"dependencies": {},
......@@ -193,13 +215,19 @@ result: dict = defaultdict(lambda: {"count": 0, "path": [],
outdated_projects = []"Get all version of numbers of cubicweb from pypi...")
cubicweb_latest_version = get_all_versions_number_from_pypi("cubicweb")[-1]
# yams_latest_version = get_all_versions_number_from_pypi("yams")[-1]
# rql_latest_version = get_all_versions_number_from_pypi("rql")[-1]
# logilab_common_latest_version = get_all_versions_number_from_pypi("logilab-common")[-1]
# logilab_database_latest_version = get_all_versions_number_from_pypi("logilab-database")[-1]"versions are: {cubicweb_latest_version}")"let's parse all artifacts..")
for gitlab_project, warnings in all_artifacts.items():"working on {} {len(warnings)} artifacts files")
for warning in warnings:
if "cubicweb" in warning.get("dependencies", ()):
current_project_cubicweb_version = tuple(map(int, warning["dependencies"]["cubicweb"].split(".")))
......@@ -238,8 +266,10 @@ for gitlab_project, warnings in all_artifacts.items():
if "full_warning" not in result[warning_text]:
result[warning_text]["full_warning"] = warning
result = sorted(result.items(), key=lambda x: -x[1]["count"])"sort result")
result = sorted(result.items(), key=lambda x: -x[1]["count"]) # type: ignore"render template")
rendered_template = jinja2.Template(open("template/deprecated-warnings.html",
......@@ -250,4 +280,6 @@ rendered_template = jinja2.Template(open("template/deprecated-warnings.html",
pygment_css=HtmlFormatter().get_style_defs())"write template to 'public/deprecated-warnings.html'")
open("public/deprecated-warnings.html", "w").write(rendered_template)"end")
......@@ -3,6 +3,10 @@
import os
import gitlab
import jinja2
import logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s:%(name)s] %(message)s")
logger = logging.getLogger("dashboard")
gl = gitlab.Gitlab("", oauth_token=os.environ.get("TOKEN"))
......@@ -10,12 +14,15 @@ all_core_projects = [x for x in gl.projects.list(all=True) if "cw-core" in x.tag
context = {"projects": [], "cubes": []}"parsing all core projects...")
for project in all_core_projects:"working on {} {project.web_url}")
# grab the pipelines for default branch that hasn't been triggered from another pipeline
pipeline_for_default = None
for pipeline in project.pipelines.list(as_list=False):"working on pipeline #{} {pipeline.web_url}")
if pipeline.ref != "branch/default":
......@@ -37,12 +44,15 @@ context["projects"] = sorted(context["projects"], key=lambda x: x["project"]["na
all_cubes = [x for x in gl.projects.list(all=True) if x.namespace["name"] == "cubes"]"parsing all cubes...")
for cube in all_cubes:"working on {} {cube.web_url}")
# grab the pipelines for default branch that hasn't been triggered from another pipeline
pipeline_for_default = None
for pipeline in cube.pipelines.list(as_list=False, all=True):"working on pipeline #{} {pipeline.web_url}")
if pipeline.ref != "branch/default":
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment