Commit 020ffdb8 authored by Laurent Peuch's avatar Laurent Peuch
Browse files

wip: deprecated-warnings dashboard

parent ba0246a4469c
......@@ -30,6 +30,16 @@ cubes_qa:
- public/qa.html
expire_in: 1 hour
deprecated_warnings:
before_script:
- pip install -r requirements.txt
script:
- TOKEN=$GITLAB_TOKEN python deprecated-warnings.py
artifacts:
paths:
- public/deprecated-warnings.html
expire_in: 1 hour
pages:
stage: deploy
only:
......
import os
import io
import json
import gitlab
import jinja2
import zipfile
from collections import defaultdict
from rdflib import ConjunctiveGraph
def download_heptapod_trig():
gl = gitlab.Gitlab('https://forge.extranet.logilab.fr', oauth_token=os.environ["GITLAB_TOKEN"])
logigraphe = gl.projects.get(426)
with open("heptapod.trig", "wb") as f:
f.write(logigraphe.artifact('branch/default', 'heptapod.trig', 'generate_dataset_for_heptapod'))
def get_all_cubes():
cnx = gitlab.Gitlab('https://forge.extranet.logilab.fr', os.environ["GITLAB_TOKEN"])
download_heptapod_trig()
g = ConjunctiveGraph()
g.parse("./heptapod.trig", format="trig")
query = """
prefix lgg: <http://data.logilab.fr/logigraphe/>
prefix lon: <http://data.logilab.fr/ontology/network/>
prefix dep: <http://ontologi.es/doap-deps#>
prefix doap: <http://usefulinc.com/ns/doap#>
SELECT ?projectId
WHERE {
?project a doap:Project .
?project doap:name ?projectName .
?project lgg:heptapod_id ?projectId .
?project lgg:has_capture_deprecated_warnings true
} ORDER BY ?projectName
"""
query_result = g.query(query)
if len(query_result) == 0:
raise Exception(f"ERROR: query failed to find any cube. Query:\n\n{query}")
for (project_id,) in query_result:
yield cnx.projects.get(id=project_id)
all_artifacts = defaultdict(list)
for cube in get_all_cubes():
print(cube.name)
branch_default_pipeline = None
for pipeline in cube.pipelines.list(as_list=False):
if pipeline.ref == "branch/default":
branch_default_pipeline = pipeline
break
else:
continue
for job in pipeline.jobs.list(all=True):
job = cube.jobs.get(job.id)
try:
artifacts_archive = zipfile.ZipFile(io.BytesIO(job.artifacts()))
except gitlab.exceptions.GitlabGetError:
# 404
continue
for artifact in artifacts_archive.filelist:
if artifact.filename.endswith("-deprecated-warnings.json"):
content = json.loads(artifacts_archive.read(artifact.filename).decode())
all_artifacts[cube.name].extend(content)
result = defaultdict(lambda: {"count": 0, "path": set(), "projects": defaultdict(int)})
for key, value in all_artifacts.items():
for warning in value:
warning_text = eval(warning["args"])[0]
result[warning_text]["count"] += warning["count"]
result[warning_text]["path"].add(f'{warning["path"].split("/site-packages/")[-1].split("/builds/cubicweb/")[-1].split("/builds/open-source/")[-1]}:{warning["lineno"]}')
result[warning_text]["projects"][key] += warning["count"]
result = sorted(result.items(), key=lambda x: -x[1]["count"])
rendered_template = jinja2.Template(open("template/deprecated-warnings.html", "r").read()).render(warnings=result)
open("public/deprecated-warnings.html", "w").write(rendered_template)
<!doctype html>
<html>
<head>
<title>Collected deprecated warnings</title>
<link href="css/style.css" rel="stylesheet" type="text/css">
</head>
<body>
<h1>Collected deprecated warnings</h1>
{% for warning_name, warning_data in warnings %}
<h4>Warning: "{{ warning_name }}"</h4>
Number of occurences: {{ warning_data["count"] }}
<br>
File where it's raised:
<ul>
{% for path in warning_data["path"]|sort %}
<li>{{ path }}</li>
{% endfor %}
</ul>
From those projects:
<ul>
{% for project, count in warning_data["projects"]|dictsort %}
<li>{{ project }}: {{ count }}</li>
{% endfor %}
</ul>
{% endfor %}
</body>
</html>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment