wip: deprecated-warnings dashboard

020ffdb8 · Laurent Peuch · ba0246a4469c · fe9685a7 · fe9685a7 · fe9685a7
Commit 020ffdb8 authored Sep 23, 2020 by Laurent Peuch
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,6 +30,16 @@ cubes_qa:
      - public/qa.html
    expire_in: 1 hour

+deprecated_warnings:
+  before_script:
+    - pip install -r requirements.txt
+  script:
+    - TOKEN=$GITLAB_TOKEN python deprecated-warnings.py
+  artifacts:
+    paths:
+      - public/deprecated-warnings.html
+    expire_in: 1 hour
+
 pages:
  stage: deploy
  only:

--- a/deprecated-warnings.py
+++ b/deprecated-warnings.py
+import os
+import io
+import json
+import gitlab
+import jinja2
+import zipfile
+
+from collections import defaultdict
+from rdflib import ConjunctiveGraph
+
+
+def download_heptapod_trig():
+    gl = gitlab.Gitlab('https://forge.extranet.logilab.fr', oauth_token=os.environ["GITLAB_TOKEN"])
+    logigraphe = gl.projects.get(426)
+
+    with open("heptapod.trig", "wb") as f:
+        f.write(logigraphe.artifact('branch/default', 'heptapod.trig', 'generate_dataset_for_heptapod'))
+
+
+def get_all_cubes():
+    cnx = gitlab.Gitlab('https://forge.extranet.logilab.fr', os.environ["GITLAB_TOKEN"])
+
+    download_heptapod_trig()
+    g = ConjunctiveGraph()
+    g.parse("./heptapod.trig", format="trig")
+
+    query = """
+    prefix lgg: <http://data.logilab.fr/logigraphe/>
+    prefix lon: <http://data.logilab.fr/ontology/network/>
+    prefix dep: <http://ontologi.es/doap-deps#>
+    prefix doap: <http://usefulinc.com/ns/doap#>
+
+    SELECT ?projectId
+
+    WHERE {
+      ?project a doap:Project .
+      ?project doap:name ?projectName .
+      ?project lgg:heptapod_id ?projectId .
+      ?project lgg:has_capture_deprecated_warnings true
+    } ORDER BY ?projectName
+    """
+
+    query_result = g.query(query)
+
+    if len(query_result) == 0:
+        raise Exception(f"ERROR: query failed to find any cube. Query:\n\n{query}")
+
+    for (project_id,) in query_result:
+        yield cnx.projects.get(id=project_id)
+
+
+all_artifacts = defaultdict(list)
+
+for cube in get_all_cubes():
+    print(cube.name)
+    branch_default_pipeline = None
+    for pipeline in cube.pipelines.list(as_list=False):
+        if pipeline.ref == "branch/default":
+            branch_default_pipeline = pipeline
+            break
+    else:
+        continue
+
+    for job in pipeline.jobs.list(all=True):
+        job = cube.jobs.get(job.id)
+        try:
+            artifacts_archive = zipfile.ZipFile(io.BytesIO(job.artifacts()))
+        except gitlab.exceptions.GitlabGetError:
+            # 404
+            continue
+
+        for artifact in artifacts_archive.filelist:
+            if artifact.filename.endswith("-deprecated-warnings.json"):
+                content = json.loads(artifacts_archive.read(artifact.filename).decode())
+                all_artifacts[cube.name].extend(content)
+
+result = defaultdict(lambda: {"count": 0, "path": set(), "projects": defaultdict(int)})
+
+for key, value in all_artifacts.items():
+    for warning in value:
+        warning_text = eval(warning["args"])[0]
+        result[warning_text]["count"] += warning["count"]
+        result[warning_text]["path"].add(f'{warning["path"].split("/site-packages/")[-1].split("/builds/cubicweb/")[-1].split("/builds/open-source/")[-1]}:{warning["lineno"]}')
+        result[warning_text]["projects"][key] += warning["count"]
+
+result = sorted(result.items(), key=lambda x: -x[1]["count"])
+
+rendered_template = jinja2.Template(open("template/deprecated-warnings.html", "r").read()).render(warnings=result)
+open("public/deprecated-warnings.html", "w").write(rendered_template)
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@ python-gitlab
 jinja2
 tox
 requests
+rdflib
--- a/template/deprecated-warnings.html
+++ b/template/deprecated-warnings.html
+<!doctype html>
+<html>
+    <head>
+        <title>Collected deprecated warnings</title>
+        <link href="css/style.css" rel="stylesheet" type="text/css">
+    </head>
+    <body>
+        <h1>Collected deprecated warnings</h1>
+
+        {% for warning_name, warning_data in warnings %}
+        <h4>Warning: "{{ warning_name }}"</h4>
+
+        Number of occurences: {{ warning_data["count"] }}
+        <br>
+        File where it's raised:
+        <ul>
+        {% for path in warning_data["path"]|sort %}
+        <li>{{ path }}</li>
+        {% endfor %}
+        </ul>
+
+        From those projects:
+        <ul>
+        {% for project, count in warning_data["projects"]|dictsort %}
+        <li>{{ project }}: {{ count }}</li>
+        {% endfor %}
+        </ul>
+
+
+        {% endfor %}
+    </body>
+</html>