Commit b710cf4d authored by François Ferry's avatar François Ferry

feat(import): import museums from csv, add zip_code to City.

close: #3
parent fe39b35b8bbe
Pipeline #28594 passed with stage
in 1 minute and 16 seconds
include *.py
include */*.py
recursive-include cubicweb_tuto *.py
recursive-include cubicweb_tuto/data *.gif *.png *.ico *.css *.js
recursive-include cubicweb_tuto/data *.gif *.png *.ico *.css *.js *.csv
recursive-include cubicweb_tuto/i18n *.po
recursive-include cubicweb_tuto/wdoc *
recursive-include test/data bootstrap_cubes *.py
......
#!/usr/bin/env python3
# coding: utf-8
from cubicweb.cwctl import CWCTL
from cubicweb.toolsutils import Command
from cubicweb.utils import admincnx
from cubicweb_tuto.dataimport import import_museums
@CWCTL.register
class ImportMuseums(Command):
"""
Import Museums and Cities from a CSV from:
https://data.culture.gouv.fr/explore/dataset/liste-et-localisation-des-musees-de-france/export/
"""
arguments = "<instance> <csv_file>"
name = "import-museums"
min_args = max_args = 2
def run(self, args):
appid, csv_file = args[:2]
with admincnx(appid) as cnx:
import_museums(cnx, csv_file)
cnx.commit()
This source diff could not be displayed because it is too large. You can view the blob instead.
""" This package allows to import data in our instance """
import csv
def import_museums(cnx, filepath):
existing_cities = dict(cnx.execute("Any Z, C Where C is City, C zip_code Z"))
existing_cities_nb = len(existing_cities)
created_museum_nb = 0
with open(filepath) as fileobj:
reader = csv.DictReader(fileobj, delimiter=";")
for record in reader:
museum_name = record["NOM DU MUSEE"]
street = record["ADR"]
zip_code = record["CP"]
city_name = record["VILLE"]
try:
lat, lng = record["coordonnees_finales"].split(",")
lat_long = {
"latitude": lat,
"longitude": lng,
}
except (AttributeError, ValueError):
lat_long = {}
try:
city = existing_cities[zip_code]
except KeyError:
city = cnx.create_entity("City", name=city_name, zip_code=zip_code)
existing_cities[zip_code] = city.eid
cnx.create_entity(
"Museum",
name=museum_name,
postal_address=f"{street}, {zip_code} {city_name}",
is_in=city,
**lat_long,
)
created_museum_nb += 1
print(
"Import finished! {} existing cities, {} cities created, {} museums created.".format(
existing_cities_nb,
len(existing_cities) - existing_cities_nb,
created_museum_nb,
)
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment