Commit 0a1e2346 authored by Adrien Di Mascio's avatar Adrien Di Mascio
Browse files

distinguish cnx and request

This eases transition to cw 3.25
parent 65b21e042ed3
......@@ -14,12 +14,12 @@ from logilab.common.shellutils import ProgressBar
from cubicweb import cwconfig, UnknownEid
from cubicweb import AuthenticationError
from cubicweb.server.utils import TasksManager
from cubicweb.utils import admincnx
from cubicweb.toolsutils import Command
from cubicweb.cwctl import CWCTL
from utils import SECTIONSPEC, extract_stats_dict, eid_from_url, \
from .utils import SECTIONSPEC, extract_stats_dict, eid_from_url, \
get_or_create_statperiod, compress_old_hits
......@@ -37,8 +37,7 @@ def url_count_from_stats(cnx, stats_dict):
for item in stats_dict['SIDER'].values():
url = item[SECTIONSPEC['SIDER'][0]]
hits = int(item[SECTIONSPEC['SIDER'][1]])
req = cnx.request()
eid = eid_from_url(req, url)
eid = eid_from_url(cnx, url)
if not eid:
continue
if 'rdf' in url:
......@@ -73,21 +72,19 @@ def track_progress(iterable, nb_ops=None, pb_size=20, pb_title=''):
class StatsUpdater(object):
def __init__(self, session, cnx, start, stop):
self.session = session
def __init__(self, cnx, start, stop):
self.cnx = cnx
self.config = session.vreg.config
self.config = cnx.vreg.config
self.start = start
self.stop = stop
self.allowed_etypes = frozenset(eschema.type for eschema in
session.vreg.schema.rschema('stats_about').objects())
cnx.vreg.schema.rschema('stats_about').objects())
self.all_hits = {}
hits_rset = session.execute('Any H,HC,HT,E,P,PSA,PSO WHERE '
hits_rset = cnx.execute('Any H,HC,HT,E,P,PSA,PSO WHERE '
'H is Hits, H count HC, H hit_type HT, '
'H stats_about E, H period P, P start PSA, P stop PSO '
'HAVING (PSA >= %(start)s, PSO <= %(stop)s) ',
{'start':start,
'stop':stop})
{'start': start, 'stop': stop})
for hit in track_progress(hits_rset.entities(), nb_ops=len(hits_rset),
pb_size=62, pb_title='Building cache'):
hit_key = (hit.stats_about[0].eid, hit.period[0].eid, hit.hit_type)
......@@ -121,7 +118,7 @@ class StatsUpdater(object):
elif periodicity == 'hour':
start = datetime(chosendate.year, chosendate.month, chosendate.day, chosendate.hour)
stop = datetime(chosendate.year, chosendate.month, chosendate.day, chosendate.hour, 59, 59)
return get_or_create_statperiod(self.cnx.request(), start, stop, stats_report)
return get_or_create_statperiod(self.cnx, start, stop, stats_report)
## update API #############################################################
def update_stats(self, skip_compress=False):
......@@ -138,9 +135,8 @@ class StatsUpdater(object):
pb_size=70, pb_title='Import'):
self._update_stats_for_date(chosendate, stats_report)
if not skip_compress:
compress_old_hits(self.session, stats_report)
self.session.commit()
self.session.set_cnxset()
compress_old_hits(self.cnx, stats_report)
self.cnx.commit()
return stats_report
def _update_stats_for_date(self, chosendate, stats_report):
......@@ -155,12 +151,11 @@ class StatsUpdater(object):
stats_report[status] += 1
def _update_hits_for_eid(self, eid, values, stats_period, hit_type):
self.session.commit()
self.session.set_cnxset()
self.cnx.commit()
visit_count = visit_count_rdf = 0
total_hits = sum([item[0] for item in values])
try:
entity = self.session.entity_from_eid(eid)
entity = self.cnx.entity_from_eid(eid)
except UnknownEid:
return 'skipped'
if entity.__regid__ not in self.allowed_etypes:
......@@ -169,8 +164,7 @@ class StatsUpdater(object):
hit = self.all_hits[(eid, stats_period.eid, hit_type)]
except KeyError: # no hit yet, create one
status = 'created'
req = self.cnx.request()
hit = req.create_entity('Hits', count=total_hits, hit_type=hit_type,
hit = self.cnx.create_entity('Hits', count=total_hits, hit_type=hit_type,
period=stats_period, stats_about=entity)
# append it to the cache
self.all_hits[(eid, stats_period.eid, hit_type)] = hit
......@@ -211,36 +205,11 @@ class UpdateWebstatsCommand(Command):
]
## command / initial setup API ############################################
def _init_cw_connection(self, appid):
from cubicweb.dbapi import in_memory_repo_cnx
config = cwconfig.instance_configuration(appid)
sourcescfg = config.sources()
config.set_sources_mode(('system',))
cnx = repo = None
while cnx is None:
try:
login = sourcescfg['admin']['login']
pwd = sourcescfg['admin']['password']
except KeyError:
login, pwd = manager_userpasswd()
try:
repo = Repository(config, TasksManager())
session = repo.new_session(login, password=pwd)
cnx = session.new_cnx()
except AuthenticationError:
print 'wrong user/password'
else:
break
# XXX keep reference on cnx otherwise cnx.__del__ will cause trouble
cnx.use_web_compatible_requests(session.vreg.config['base-url'])
return cnx, session
def run(self, args):
# args = (appid, start[, stop])
appid = args.pop(0)
cw_cnx, session = self._init_cw_connection(appid)
session.set_cnxset()
periodicity = session.vreg.config['awstats-periodicity']
with admincnx(appid) as cnx:
periodicity = cnx.vreg.config['awstats-periodicity']
start = stop = None
if len(args) > 0:
start = parse_input_date(args[0], periodicity)
......@@ -257,7 +226,7 @@ class UpdateWebstatsCommand(Command):
if start is None or stop is None:
sys.exit(1) # parse_input_date failed to parse date
stop += ONEDAY # date_range() excludes stop boundary
stats_updater = StatsUpdater(session, cw_cnx, start, stop)
stats_updater = StatsUpdater(cnx, start, stop)
stats_report = stats_updater.update_stats(self.config.skip_compress)
print '''=== Update Report ===
Number of periods imported : %(periods)s
......
......@@ -27,6 +27,7 @@ except ImportError:
_ = unicode
from cubicweb.req import FindEntityError
from cubicweb.web import NotFound, Redirect
from cubicweb.devtools.fake import FakeRequest
from psycopg2 import DataError
......@@ -142,26 +143,27 @@ def extract_stats_dict(filepath):
section_name = None
return stats_dict
def eid_from_url(req, value):
def eid_from_url(cnx, value):
''' return an eid from an url '''
url_resolver = req.vreg['components'].select('urlpublisher',
vreg=req.vreg)
req.url = lambda includeparams: value
req.relative_path = lambda includeparams: value[1:]
req = FakeRequest(vreg=cnx.vreg, url=value)
req.set_cnx(cnx)
url_resolver = cnx.vreg['components'].select('urlpublisher',
vreg=cnx.vreg)
try:
pmid, rset = url_resolver.process(req, value)
if rset and len(rset) == 1:
return rset[0][0]
elif req.form.get('rql'):
rset = req.execute(req.form.get('rql'))
rset = cnx.execute(req.form.get('rql'))
if rset and len(rset) == 1:
return rset[0][0]
except (NotFound, DataError, Redirect):
pass
def get_or_create_statperiod(session, start, stop, stats_report={}):
def get_or_create_statperiod(cnx, start, stop, stats_report={}):
rql = 'Any P WHERE P is StatPeriod, P start "%(start_date)s", P stop "%(end_date)s"'
rset = session.execute(rql %
rset = cnx.execute(rql %
{'start_date':start,
'end_date':stop})
if rset:
......@@ -169,11 +171,11 @@ def get_or_create_statperiod(session, start, stop, stats_report={}):
else:
stats_report.setdefault('periods', 0)
stats_report['periods'] += 1
return session.create_entity('StatPeriod', start=start, stop=stop)
return cnx.create_entity('StatPeriod', start=start, stop=stop)
def time_params(req):
def time_params(cnx):
params = []
rset = req.execute('Any START ORDERBY START LIMIT 1 WHERE P is StatPeriod, P start START, P stop STOP HAVING STOP-START <= 2')
rset = cnx.execute('Any START ORDERBY START LIMIT 1 WHERE P is StatPeriod, P start START, P stop STOP HAVING STOP-START <= 2')
for (item,) in rset:
for first_day in date_range(previous_month(item), previous_month(datetime.now(), 5), incmonth=True):
delta = 2
......@@ -182,8 +184,8 @@ def time_params(req):
return params
def compress_old_hits(req, update_stats={}, progressbar=True):
tp = time_params(req)
def compress_old_hits(cnx, update_stats={}, progressbar=True):
tp = time_params(cnx)
if progressbar:
pb = ProgressBar(len(tp), 55, title='Compressing old stats')
for start, stop, delta in tp:
......@@ -193,9 +195,9 @@ def compress_old_hits(req, update_stats={}, progressbar=True):
'H period P, P start >= %(start)s, P stop <= %(stop)s, H stats_about E,'\
'P start START, P stop STOP HAVING STOP-START <= %(timedelta)s'
results = {}
type_rset = req.execute('DISTINCT Any C WHERE X is Hits, X hit_type C')
type_rset = cnx.execute('DISTINCT Any C WHERE X is Hits, X hit_type C')
for (hit_type,) in type_rset:
results[hit_type] = req.execute(rql,
results[hit_type] = cnx.execute(rql,
{'start': start,
'stop': stop,
'hit_type':hit_type,
......@@ -203,16 +205,16 @@ def compress_old_hits(req, update_stats={}, progressbar=True):
if not any(results.values()):
continue
# deleting statperiod deletes all associated hits
drset = req.execute('DELETE StatPeriod P WHERE P start >= %(start)s, P stop <= %(stop)s',
drset = cnx.execute('DELETE StatPeriod P WHERE P start >= %(start)s, P stop <= %(stop)s',
{'start': start,
'stop': stop,})
update_stats['compressed'] += len(drset)
stp = get_or_create_statperiod(req, start, stop)
stp = get_or_create_statperiod(cnx, start, stop)
for hit_type, rset in results.items():
for eid, count in rset:
content_entity = req.entity_from_eid(eid)
content_entity = cnx.entity_from_eid(eid)
# FIXME if Hits for period and content exist, update it ?
req.create_entity('Hits', hit_type=hit_type, period=stp, count=count,
cnx.create_entity('Hits', hit_type=hit_type, period=stp, count=count,
stats_about=content_entity)
if progressbar:
pb.finish()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment