Commit 03091337 authored by Vincent Michel's avatar Vincent Michel
Browse files

[ccplugin] Add import command

parent bf6850f1d36c
import os
import os.path as osp
import time
import glob
import csv
from datetime import date, datetime
from cubicweb import AuthenticationError
from cubicweb import cwconfig
from cubicweb.server.utils import manager_userpasswd
from cubicweb.dbapi import in_memory_repo_cnx
from cubicweb.toolsutils import Command
from cubicweb.cwctl import CWCTL
import cubicweb.dataimport as cwdi
class BibliovizImportCommand(Command):
"""
Command for importing biblioviz data.
"""
arguments = '<instance> <dumpname>'
max_args = 2
name = 'biblioviz-import'
def _init_cw_connection(self, appid):
config = cwconfig.instance_configuration(appid)
sourcescfg = config.sources()
config.set_sources_mode(('system',))
cnx = repo = None
while cnx is None:
try:
login = sourcescfg['admin']['login']
pwd = sourcescfg['admin']['password']
except KeyError:
login, pwd = manager_userpasswd()
try:
repo, cnx = in_memory_repo_cnx(config, login=login, password=pwd)
except AuthenticationError:
print 'wrong user/password'
else:
break
session = repo._get_session(cnx.sessionid)
return cnx, session
def run(self, args):
appid = args.pop(0)
dumpname = args[0] if len(args) > 0 else None
if not osp.exists(dumpname):
print 'Folder %s does not exists' % dumpname
cw_cnx, session = self._init_cw_connection(appid)
session.set_pool()
start_time = time.time()
store = cwdi.SQLGenObjectStore(session)
datetime_format='%Y-%m-%d'
portails = {}
for filename in glob.glob('%s/*.csv' % dumpname):
if not osp.exists(filename):
print 'File %s does not exist in folder %s' % (filename, dumpname)
continue
print 'Processing %s' % filename
csvreader = csv.reader(open(filename), delimiter=';')
# Skip headline
csvreader.next()
for row in csvreader:
if len(row) == 5:
site, date, heure, nom, entrees = row
pays = u'France'
else:
site, pays, date, heure, nom, entrees = row
date = datetime.strptime(date, datetime_format)
mois = date.month
heure = int(heure.split(':')[0])
retour = nom.endswith('retour')
entrees = int(entrees)
# Problem with some names
if nom.endswith('vie du citoyen'):
nom = u'1er etage vie du citoyen'
# Portail
if nom not in portails:
portail = store.create_entity('Portail', nom=nom, site=site,
pays=pays, retour=retour)
portail_eid = portail.eid
portails[nom] = portail_eid
else:
portail_eid = portails[nom]
# Decompte
decompte = store.create_entity('Decompte', date=date, mois=mois,
heure=heure, entrees=entrees, portail=portail_eid)
store.flush()
print 'time', time.time() - start_time, (time.time() - start_time)/60.
CWCTL.register(BibliovizImportCommand)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment