Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
cubicweb
dashboards
Commits
fe9685a75f5a
Commit
020ffdb8
authored
Sep 23, 2020
by
Laurent Peuch
Browse files
wip: deprecated-warnings dashboard
parent
ba0246a4469c
Changes
4
Hide whitespace changes
Inline
Side-by-side
.gitlab-ci.yml
View file @
fe9685a7
...
...
@@ -30,6 +30,16 @@ cubes_qa:
-
public/qa.html
expire_in
:
1 hour
deprecated_warnings
:
before_script
:
-
pip install -r requirements.txt
script
:
-
TOKEN=$GITLAB_TOKEN python deprecated-warnings.py
artifacts
:
paths
:
-
public/deprecated-warnings.html
expire_in
:
1 hour
pages
:
stage
:
deploy
only
:
...
...
deprecated-warnings.py
0 → 100644
View file @
fe9685a7
import
os
import
io
import
json
import
gitlab
import
jinja2
import
zipfile
from
collections
import
defaultdict
from
rdflib
import
ConjunctiveGraph
def
download_heptapod_trig
():
gl
=
gitlab
.
Gitlab
(
'https://forge.extranet.logilab.fr'
,
oauth_token
=
os
.
environ
[
"GITLAB_TOKEN"
])
logigraphe
=
gl
.
projects
.
get
(
426
)
with
open
(
"heptapod.trig"
,
"wb"
)
as
f
:
f
.
write
(
logigraphe
.
artifact
(
'branch/default'
,
'heptapod.trig'
,
'generate_dataset_for_heptapod'
))
def
get_all_cubes
():
cnx
=
gitlab
.
Gitlab
(
'https://forge.extranet.logilab.fr'
,
os
.
environ
[
"GITLAB_TOKEN"
])
download_heptapod_trig
()
g
=
ConjunctiveGraph
()
g
.
parse
(
"./heptapod.trig"
,
format
=
"trig"
)
query
=
"""
prefix lgg: <http://data.logilab.fr/logigraphe/>
prefix lon: <http://data.logilab.fr/ontology/network/>
prefix dep: <http://ontologi.es/doap-deps#>
prefix doap: <http://usefulinc.com/ns/doap#>
SELECT ?projectId
WHERE {
?project a doap:Project .
?project doap:name ?projectName .
?project lgg:heptapod_id ?projectId .
?project lgg:has_capture_deprecated_warnings true
} ORDER BY ?projectName
"""
query_result
=
g
.
query
(
query
)
if
len
(
query_result
)
==
0
:
raise
Exception
(
f
"ERROR: query failed to find any cube. Query:
\n\n
{
query
}
"
)
for
(
project_id
,)
in
query_result
:
yield
cnx
.
projects
.
get
(
id
=
project_id
)
all_artifacts
=
defaultdict
(
list
)
for
cube
in
get_all_cubes
():
print
(
cube
.
name
)
branch_default_pipeline
=
None
for
pipeline
in
cube
.
pipelines
.
list
(
as_list
=
False
):
if
pipeline
.
ref
==
"branch/default"
:
branch_default_pipeline
=
pipeline
break
else
:
continue
for
job
in
pipeline
.
jobs
.
list
(
all
=
True
):
job
=
cube
.
jobs
.
get
(
job
.
id
)
try
:
artifacts_archive
=
zipfile
.
ZipFile
(
io
.
BytesIO
(
job
.
artifacts
()))
except
gitlab
.
exceptions
.
GitlabGetError
:
# 404
continue
for
artifact
in
artifacts_archive
.
filelist
:
if
artifact
.
filename
.
endswith
(
"-deprecated-warnings.json"
):
content
=
json
.
loads
(
artifacts_archive
.
read
(
artifact
.
filename
).
decode
())
all_artifacts
[
cube
.
name
].
extend
(
content
)
result
=
defaultdict
(
lambda
:
{
"count"
:
0
,
"path"
:
set
(),
"projects"
:
defaultdict
(
int
)})
for
key
,
value
in
all_artifacts
.
items
():
for
warning
in
value
:
warning_text
=
eval
(
warning
[
"args"
])[
0
]
result
[
warning_text
][
"count"
]
+=
warning
[
"count"
]
result
[
warning_text
][
"path"
].
add
(
f
'
{
warning
[
"path"
].
split
(
"/site-packages/"
)[
-
1
].
split
(
"/builds/cubicweb/"
)[
-
1
].
split
(
"/builds/open-source/"
)[
-
1
]
}
:
{
warning
[
"lineno"
]
}
'
)
result
[
warning_text
][
"projects"
][
key
]
+=
warning
[
"count"
]
result
=
sorted
(
result
.
items
(),
key
=
lambda
x
:
-
x
[
1
][
"count"
])
rendered_template
=
jinja2
.
Template
(
open
(
"template/deprecated-warnings.html"
,
"r"
).
read
()).
render
(
warnings
=
result
)
open
(
"public/deprecated-warnings.html"
,
"w"
).
write
(
rendered_template
)
requirements.txt
View file @
fe9685a7
...
...
@@ -2,3 +2,4 @@ python-gitlab
jinja2
tox
requests
rdflib
template/deprecated-warnings.html
0 → 100644
View file @
fe9685a7
<!doctype html>
<html>
<head>
<title>
Collected deprecated warnings
</title>
<link
href=
"css/style.css"
rel=
"stylesheet"
type=
"text/css"
>
</head>
<body>
<h1>
Collected deprecated warnings
</h1>
{% for warning_name, warning_data in warnings %}
<h4>
Warning: "{{ warning_name }}"
</h4>
Number of occurences: {{ warning_data["count"] }}
<br>
File where it's raised:
<ul>
{% for path in warning_data["path"]|sort %}
<li>
{{ path }}
</li>
{% endfor %}
</ul>
From those projects:
<ul>
{% for project, count in warning_data["projects"]|dictsort %}
<li>
{{ project }}: {{ count }}
</li>
{% endfor %}
</ul>
{% endfor %}
</body>
</html>
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment