deprecated-warnings.py 5.91 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
import os
import io
import json
import gitlab
import jinja2
import zipfile

from collections import defaultdict
from rdflib import ConjunctiveGraph

11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters.html import HtmlFormatter


def highlight_code(file_content, lineno):
    file_content = file_content.split("\n")[:lineno + 4]

    # si on est à la ligne 3
    # d'un fichier de 4 lignes
    if len(file_content) > lineno:
        hl_lines = [5]
        file_content = file_content[lineno - 5:]
        linenostart = lineno - 4
    else:
        hl_lines = [lineno]
        linenostart = 1

    file_content = "\n".join(file_content)

    return highlight(file_content, get_lexer_by_name("python", stripnl=False),
                     HtmlFormatter(wrapcode=True, linenos=True,
                                   linenostart=linenostart, hl_lines=hl_lines))

35
36
37
38
39

def download_heptapod_trig():
    gl = gitlab.Gitlab('https://forge.extranet.logilab.fr', oauth_token=os.environ["GITLAB_TOKEN"])
    logigraphe = gl.projects.get(426)

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
    for pipeline in logigraphe.pipelines.list(as_list=False):
        if pipeline.ref != "branch/default":
            continue

        job = [x for x in pipeline.jobs.list(all=True) if x.name == "generate_dataset_for_heptapod"]

        if not job:
            continue

        job = logigraphe.jobs.get(job[0].id)
        try:
            artifacts_archive = zipfile.ZipFile(io.BytesIO(job.artifacts()))
        except gitlab.exceptions.GitlabGetError:
            # 404
            continue

        heptapod_trig = [x for x in artifacts_archive.filelist if x.filename == "heptapod.trig"]
        if not heptapod_trig:
            continue

        content = artifacts_archive.read("heptapod.trig")
        if not content:
            continue

        with open("heptapod.trig", "wb") as f:
            f.write(content)

        return

    raise Exception("failed to find heptapod.trig")
70
71


72
73
74
75
76
77
78
79
def download_files_to_url_cache():
    gl = gitlab.Gitlab('https://forge.extranet.logilab.fr', oauth_token=os.environ["GITLAB_TOKEN"])
    logigraphe = gl.projects.get(526)

    with open("files_to_url_cache.json", "wb") as f:
        f.write(logigraphe.artifact('branch/default', 'files_to_url_cache.json', 'build_cache'))


80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
def get_all_cubes():
    cnx = gitlab.Gitlab('https://forge.extranet.logilab.fr', os.environ["GITLAB_TOKEN"])

    download_heptapod_trig()
    g = ConjunctiveGraph()
    g.parse("./heptapod.trig", format="trig")

    query = """
    prefix lgg: <http://data.logilab.fr/logigraphe/>
    prefix lon: <http://data.logilab.fr/ontology/network/>
    prefix dep: <http://ontologi.es/doap-deps#>
    prefix doap: <http://usefulinc.com/ns/doap#>

    SELECT ?projectId

    WHERE {
      ?project a doap:Project .
      ?project doap:name ?projectName .
      ?project lgg:heptapod_id ?projectId .
      ?project lgg:has_capture_deprecated_warnings true
    } ORDER BY ?projectName
    """

    query_result = g.query(query)

    if len(query_result) == 0:
        raise Exception(f"ERROR: query failed to find any cube. Query:\n\n{query}")

    for (project_id,) in query_result:
        yield cnx.projects.get(id=project_id)


112
113
114
download_files_to_url_cache()
files_to_url_cache = json.load(open("files_to_url_cache.json"))

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
all_artifacts = defaultdict(list)

for cube in get_all_cubes():
    print(cube.name)
    branch_default_pipeline = None
    for pipeline in cube.pipelines.list(as_list=False):
        if pipeline.ref == "branch/default":
            branch_default_pipeline = pipeline
            break
    else:
        continue

    for job in pipeline.jobs.list(all=True):
        job = cube.jobs.get(job.id)
        try:
            artifacts_archive = zipfile.ZipFile(io.BytesIO(job.artifacts()))
        except gitlab.exceptions.GitlabGetError:
            # 404
            continue

        for artifact in artifacts_archive.filelist:
136
            if artifact.filename.endswith("deprecated-warnings.json"):
137
138
139
140
141
                content = artifacts_archive.read(artifact.filename).decode()
                if not content:
                    continue

                content = json.loads(content)
142
143
                all_artifacts[cube.name].extend(content)

144
result: dict = defaultdict(lambda: {"count": 0, "path": [], "projects": defaultdict(int)})
145
146
147
148
149
150
151

for key, value in all_artifacts.items():
    for warning in value:
        warning_text = eval(warning["args"])[0]
        result[warning_text]["count"] += warning["count"]
        result[warning_text]["projects"][key] += warning["count"]

152
153
154
        file_name = warning["path"].split("/site-packages/")[-1].split("/builds/cubicweb/")[-1].split("/builds/open-source/")[-1]

        if file_name not in {x[0] for x in result[warning_text]["path"]}:
155
            result[warning_text]["path"].append([f'{file_name}',
156
                                                 warning["lineno"],
157
158
159
160
161
162
163
164
165
166
                                                 warning.get("file_content")])
        elif warning.get("file_content"):
            for warning_content in result[warning_text]["path"]:
                this_warning_file_name, _, file_content = warning_content

                if this_warning_file_name != file_name:
                    continue

                if not file_content:
                    warning_content[2] = warning["file_content"]
167
168
169

    result[warning_text]["path"] = sorted(result[warning_text]["path"], key=lambda x: x[0])

170
171
result = sorted(result.items(), key=lambda x: -x[1]["count"])

172
173
174
175
176
rendered_template = jinja2.Template(open("template/deprecated-warnings.html",
                                         "r").read()).render(warnings=result,
                                                             files_to_url=files_to_url_cache,
                                                             highlight_code=highlight_code,
                                                             pygment_css=HtmlFormatter().get_style_defs())
177
open("public/deprecated-warnings.html", "w").write(rendered_template)