deprecated-warnings.py 9.24 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
import os
import io
import json
import gitlab
import jinja2
import zipfile

from collections import defaultdict
from rdflib import ConjunctiveGraph

11
12
13
14
15
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters.html import HtmlFormatter


16
17
18
def highlight_code(file_content, lineno, full_file=False):
    if full_file:
        return highlight(file_content, get_lexer_by_name("python", stripnl=False),
19
20
                         HtmlFormatter(wrapcode=True, linenos=True, lineanchors="L",
                                       anchorlinenos=True, hl_lines=[lineno]))
21

22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
    file_content = file_content.split("\n")[:lineno + 4]

    # si on est à la ligne 3
    # d'un fichier de 4 lignes
    if len(file_content) > lineno:
        hl_lines = [5]
        file_content = file_content[lineno - 5:]
        linenostart = lineno - 4
    else:
        hl_lines = [lineno]
        linenostart = 1

    file_content = "\n".join(file_content)

    return highlight(file_content, get_lexer_by_name("python", stripnl=False),
                     HtmlFormatter(wrapcode=True, linenos=True,
                                   linenostart=linenostart, hl_lines=hl_lines))

40
41
42
43
44

def download_heptapod_trig():
    gl = gitlab.Gitlab('https://forge.extranet.logilab.fr', oauth_token=os.environ["GITLAB_TOKEN"])
    logigraphe = gl.projects.get(426)

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
    for pipeline in logigraphe.pipelines.list(as_list=False):
        if pipeline.ref != "branch/default":
            continue

        job = [x for x in pipeline.jobs.list(all=True) if x.name == "generate_dataset_for_heptapod"]

        if not job:
            continue

        job = logigraphe.jobs.get(job[0].id)
        try:
            artifacts_archive = zipfile.ZipFile(io.BytesIO(job.artifacts()))
        except gitlab.exceptions.GitlabGetError:
            # 404
            continue

        heptapod_trig = [x for x in artifacts_archive.filelist if x.filename == "heptapod.trig"]
        if not heptapod_trig:
            continue

        content = artifacts_archive.read("heptapod.trig")
        if not content:
            continue

        with open("heptapod.trig", "wb") as f:
            f.write(content)

        return

    raise Exception("failed to find heptapod.trig")
75
76


77
78
79
80
81
82
83
84
def download_files_to_url_cache():
    gl = gitlab.Gitlab('https://forge.extranet.logilab.fr', oauth_token=os.environ["GITLAB_TOKEN"])
    logigraphe = gl.projects.get(526)

    with open("files_to_url_cache.json", "wb") as f:
        f.write(logigraphe.artifact('branch/default', 'files_to_url_cache.json', 'build_cache'))


85
86
87
88
89
90
91
92
93
94
95
96
97
def get_all_cubes():
    cnx = gitlab.Gitlab('https://forge.extranet.logilab.fr', os.environ["GITLAB_TOKEN"])

    download_heptapod_trig()
    g = ConjunctiveGraph()
    g.parse("./heptapod.trig", format="trig")

    query = """
    prefix lgg: <http://data.logilab.fr/logigraphe/>
    prefix lon: <http://data.logilab.fr/ontology/network/>
    prefix dep: <http://ontologi.es/doap-deps#>
    prefix doap: <http://usefulinc.com/ns/doap#>

98
    SELECT ?projectId ?gitlabTag
99
100
101
102
103

    WHERE {
      ?project a doap:Project .
      ?project doap:name ?projectName .
      ?project lgg:heptapod_id ?projectId .
104
      ?project doap:category ?gitlabTag .
105
106
107
108
109
110
111
112
113
      ?project lgg:has_capture_deprecated_warnings true
    } ORDER BY ?projectName
    """

    query_result = g.query(query)

    if len(query_result) == 0:
        raise Exception(f"ERROR: query failed to find any cube. Query:\n\n{query}")

114
115
116
    for (project_id, tag) in query_result:
        if str(tag) not in ("cw-core", "project-dependency"):
            continue
117
118
119
        yield cnx.projects.get(id=project_id)


120
def render_traceback(traceback_id, traceback, warning_name, project_id):
121
122
123
124
125
126
    os.makedirs("public/traceback", exist_ok=True)

    rendered_template = jinja2.Template(open("template/traceback.html",
                                             "r").read()).render(traceback=reversed(traceback),
                                                                 files_to_url=files_to_url_cache,
                                                                 highlight_code=highlight_code,
127
                                                                 render_source_code_file=render_source_code_file,
128
                                                                 warning_name=warning_name,
129
130
                                                                 str=str,
                                                                 project_id=project_id,
131
132
133
134
                                                                 pygment_css=HtmlFormatter().get_style_defs())
    open(f"public/traceback/{traceback_id}.html", "w").write(rendered_template)


135
136
137
ALREADY_GENERATED_FILES = set()


138
def render_source_code_file(file_content, path, file_name, lineno):
139
140
141
    if path in ALREADY_GENERATED_FILES:
        return

142
143
144
145
146
147
148
149
150
151
    os.makedirs("public/file", exist_ok=True)

    rendered_template = jinja2.Template(open("template/file.html",
                                             "r").read()).render(file_content=file_content,
                                                                 files_to_url=files_to_url_cache,
                                                                 highlight_code=highlight_code,
                                                                 file_name=file_name,
                                                                 lineno=lineno,
                                                                 pygment_css=HtmlFormatter().get_style_defs())
    open(f"public/file/{path}.html", "w").write(rendered_template)
152
    ALREADY_GENERATED_FILES.add(path)
153
154


155
156
157
download_files_to_url_cache()
files_to_url_cache = json.load(open("files_to_url_cache.json"))

158
159
160
161
162
163
all_artifacts = defaultdict(list)

for cube in get_all_cubes():
    print(cube.name)
    branch_default_pipeline = None
    for pipeline in cube.pipelines.list(as_list=False):
164
        if pipeline.ref == "branch/default" and pipeline.status == "success":
165
166
167
168
169
170
171
172
173
174
175
176
177
178
            branch_default_pipeline = pipeline
            break
    else:
        continue

    for job in pipeline.jobs.list(all=True):
        job = cube.jobs.get(job.id)
        try:
            artifacts_archive = zipfile.ZipFile(io.BytesIO(job.artifacts()))
        except gitlab.exceptions.GitlabGetError:
            # 404
            continue

        for artifact in artifacts_archive.filelist:
179
            if artifact.filename.endswith("deprecated-warnings.json"):
180
181
182
183
184
                content = artifacts_archive.read(artifact.filename).decode()
                if not content:
                    continue

                content = json.loads(content)
185
                all_artifacts[cube].extend(content)
186

187
188
result: dict = defaultdict(lambda: {"count": 0, "path": [],
                                    "projects": defaultdict(lambda: {"count": 0,
189
190
                                                                     "dependencies": {},
                                                                     "warnings": {}})})
191

192
193
for gitlab_project, warnings in all_artifacts.items():
    for warning in warnings:
194
195
        warning_text = eval(warning["args"])[0]
        result[warning_text]["count"] += warning["count"]
196
        result[warning_text]["projects"][gitlab_project]["count"] += warning["count"]
197

198
199
        if "dependencies" in warning and not result[warning_text]["projects"][gitlab_project]["dependencies"]:
            result[warning_text]["projects"][gitlab_project]["dependencies"] = warning["dependencies"]
200

201
        warning_key = (warning["filename"], warning["lineno"])
202

203
204
205
        warning_key = (warning["filename"], warning["lineno"])
        if warning_key not in result[warning_text]["projects"][gitlab_project]["warnings"]:
            result[warning_text]["projects"][gitlab_project]["warnings"][warning_key] = warning
206

207
208
209
210
211
212
213
        try:
            gitlab_project.files.get(warning["filename"], ref="branch/default")
        except gitlab.exceptions.GitlabGetError:
            warning["file_in_current_repo"] = False
        else:
            warning["file_in_current_repo"] = True

214
        # this is done to display a summary of the warning at the top
215
216
217
218
        if warning["category"] != "DeprecationWarning":
            if "test_file" in warning and "/site-packages/" in warning["test_file"]:
                warning["test_file"] = warning["test_file"].split("/site-packages/")[1]

219
            result[warning_text]["projects"][gitlab_project]["full_warning"] = warning
220
221
222

            if "full_warning" not in result[warning_text]:
                result[warning_text]["full_warning"] = warning
223

224
225
result = sorted(result.items(), key=lambda x: -x[1]["count"])

226
227
228
229
rendered_template = jinja2.Template(open("template/deprecated-warnings.html",
                                         "r").read()).render(warnings=result,
                                                             files_to_url=files_to_url_cache,
                                                             highlight_code=highlight_code,
230
231
                                                             str=str,
                                                             render_traceback=render_traceback,
232
                                                             render_source_code_file=render_source_code_file,
233
                                                             pygment_css=HtmlFormatter().get_style_defs())
234
open("public/deprecated-warnings.html", "w").write(rendered_template)