Commit 0a1e2346 authored by Adrien Di Mascio's avatar Adrien Di Mascio
Browse files

distinguish cnx and request

This eases transition to cw 3.25
parent 65b21e042ed3
...@@ -14,12 +14,12 @@ from logilab.common.shellutils import ProgressBar ...@@ -14,12 +14,12 @@ from logilab.common.shellutils import ProgressBar
from cubicweb import cwconfig, UnknownEid from cubicweb import cwconfig, UnknownEid
from cubicweb import AuthenticationError from cubicweb import AuthenticationError
from cubicweb.server.utils import TasksManager from cubicweb.utils import admincnx
from cubicweb.toolsutils import Command from cubicweb.toolsutils import Command
from cubicweb.cwctl import CWCTL from cubicweb.cwctl import CWCTL
from utils import SECTIONSPEC, extract_stats_dict, eid_from_url, \ from .utils import SECTIONSPEC, extract_stats_dict, eid_from_url, \
get_or_create_statperiod, compress_old_hits get_or_create_statperiod, compress_old_hits
...@@ -37,8 +37,7 @@ def url_count_from_stats(cnx, stats_dict): ...@@ -37,8 +37,7 @@ def url_count_from_stats(cnx, stats_dict):
for item in stats_dict['SIDER'].values(): for item in stats_dict['SIDER'].values():
url = item[SECTIONSPEC['SIDER'][0]] url = item[SECTIONSPEC['SIDER'][0]]
hits = int(item[SECTIONSPEC['SIDER'][1]]) hits = int(item[SECTIONSPEC['SIDER'][1]])
req = cnx.request() eid = eid_from_url(cnx, url)
eid = eid_from_url(req, url)
if not eid: if not eid:
continue continue
if 'rdf' in url: if 'rdf' in url:
...@@ -73,21 +72,19 @@ def track_progress(iterable, nb_ops=None, pb_size=20, pb_title=''): ...@@ -73,21 +72,19 @@ def track_progress(iterable, nb_ops=None, pb_size=20, pb_title=''):
class StatsUpdater(object): class StatsUpdater(object):
def __init__(self, session, cnx, start, stop): def __init__(self, cnx, start, stop):
self.session = session
self.cnx = cnx self.cnx = cnx
self.config = session.vreg.config self.config = cnx.vreg.config
self.start = start self.start = start
self.stop = stop self.stop = stop
self.allowed_etypes = frozenset(eschema.type for eschema in self.allowed_etypes = frozenset(eschema.type for eschema in
session.vreg.schema.rschema('stats_about').objects()) cnx.vreg.schema.rschema('stats_about').objects())
self.all_hits = {} self.all_hits = {}
hits_rset = session.execute('Any H,HC,HT,E,P,PSA,PSO WHERE ' hits_rset = cnx.execute('Any H,HC,HT,E,P,PSA,PSO WHERE '
'H is Hits, H count HC, H hit_type HT, ' 'H is Hits, H count HC, H hit_type HT, '
'H stats_about E, H period P, P start PSA, P stop PSO ' 'H stats_about E, H period P, P start PSA, P stop PSO '
'HAVING (PSA >= %(start)s, PSO <= %(stop)s) ', 'HAVING (PSA >= %(start)s, PSO <= %(stop)s) ',
{'start':start, {'start': start, 'stop': stop})
'stop':stop})
for hit in track_progress(hits_rset.entities(), nb_ops=len(hits_rset), for hit in track_progress(hits_rset.entities(), nb_ops=len(hits_rset),
pb_size=62, pb_title='Building cache'): pb_size=62, pb_title='Building cache'):
hit_key = (hit.stats_about[0].eid, hit.period[0].eid, hit.hit_type) hit_key = (hit.stats_about[0].eid, hit.period[0].eid, hit.hit_type)
...@@ -121,7 +118,7 @@ class StatsUpdater(object): ...@@ -121,7 +118,7 @@ class StatsUpdater(object):
elif periodicity == 'hour': elif periodicity == 'hour':
start = datetime(chosendate.year, chosendate.month, chosendate.day, chosendate.hour) start = datetime(chosendate.year, chosendate.month, chosendate.day, chosendate.hour)
stop = datetime(chosendate.year, chosendate.month, chosendate.day, chosendate.hour, 59, 59) stop = datetime(chosendate.year, chosendate.month, chosendate.day, chosendate.hour, 59, 59)
return get_or_create_statperiod(self.cnx.request(), start, stop, stats_report) return get_or_create_statperiod(self.cnx, start, stop, stats_report)
## update API ############################################################# ## update API #############################################################
def update_stats(self, skip_compress=False): def update_stats(self, skip_compress=False):
...@@ -138,9 +135,8 @@ class StatsUpdater(object): ...@@ -138,9 +135,8 @@ class StatsUpdater(object):
pb_size=70, pb_title='Import'): pb_size=70, pb_title='Import'):
self._update_stats_for_date(chosendate, stats_report) self._update_stats_for_date(chosendate, stats_report)
if not skip_compress: if not skip_compress:
compress_old_hits(self.session, stats_report) compress_old_hits(self.cnx, stats_report)
self.session.commit() self.cnx.commit()
self.session.set_cnxset()
return stats_report return stats_report
def _update_stats_for_date(self, chosendate, stats_report): def _update_stats_for_date(self, chosendate, stats_report):
...@@ -155,12 +151,11 @@ class StatsUpdater(object): ...@@ -155,12 +151,11 @@ class StatsUpdater(object):
stats_report[status] += 1 stats_report[status] += 1
def _update_hits_for_eid(self, eid, values, stats_period, hit_type): def _update_hits_for_eid(self, eid, values, stats_period, hit_type):
self.session.commit() self.cnx.commit()
self.session.set_cnxset()
visit_count = visit_count_rdf = 0 visit_count = visit_count_rdf = 0
total_hits = sum([item[0] for item in values]) total_hits = sum([item[0] for item in values])
try: try:
entity = self.session.entity_from_eid(eid) entity = self.cnx.entity_from_eid(eid)
except UnknownEid: except UnknownEid:
return 'skipped' return 'skipped'
if entity.__regid__ not in self.allowed_etypes: if entity.__regid__ not in self.allowed_etypes:
...@@ -169,9 +164,8 @@ class StatsUpdater(object): ...@@ -169,9 +164,8 @@ class StatsUpdater(object):
hit = self.all_hits[(eid, stats_period.eid, hit_type)] hit = self.all_hits[(eid, stats_period.eid, hit_type)]
except KeyError: # no hit yet, create one except KeyError: # no hit yet, create one
status = 'created' status = 'created'
req = self.cnx.request() hit = self.cnx.create_entity('Hits', count=total_hits, hit_type=hit_type,
hit = req.create_entity('Hits', count=total_hits, hit_type=hit_type, period=stats_period, stats_about=entity)
period=stats_period, stats_about=entity)
# append it to the cache # append it to the cache
self.all_hits[(eid, stats_period.eid, hit_type)] = hit self.all_hits[(eid, stats_period.eid, hit_type)] = hit
else: else:
...@@ -211,55 +205,30 @@ class UpdateWebstatsCommand(Command): ...@@ -211,55 +205,30 @@ class UpdateWebstatsCommand(Command):
] ]
## command / initial setup API ############################################ ## command / initial setup API ############################################
def _init_cw_connection(self, appid):
from cubicweb.dbapi import in_memory_repo_cnx
config = cwconfig.instance_configuration(appid)
sourcescfg = config.sources()
config.set_sources_mode(('system',))
cnx = repo = None
while cnx is None:
try:
login = sourcescfg['admin']['login']
pwd = sourcescfg['admin']['password']
except KeyError:
login, pwd = manager_userpasswd()
try:
repo = Repository(config, TasksManager())
session = repo.new_session(login, password=pwd)
cnx = session.new_cnx()
except AuthenticationError:
print 'wrong user/password'
else:
break
# XXX keep reference on cnx otherwise cnx.__del__ will cause trouble
cnx.use_web_compatible_requests(session.vreg.config['base-url'])
return cnx, session
def run(self, args): def run(self, args):
# args = (appid, start[, stop]) # args = (appid, start[, stop])
appid = args.pop(0) appid = args.pop(0)
cw_cnx, session = self._init_cw_connection(appid) with admincnx(appid) as cnx:
session.set_cnxset() periodicity = cnx.vreg.config['awstats-periodicity']
periodicity = session.vreg.config['awstats-periodicity'] start = stop = None
start = stop = None if len(args) > 0:
if len(args) > 0: start = parse_input_date(args[0], periodicity)
start = parse_input_date(args[0], periodicity) if start is None:
if start is None: if self.config.today:
if self.config.today: chosendate = datetime.now()
chosendate = datetime.now() else:
else: chosendate = datetime.now()-timedelta(1)
chosendate = datetime.now()-timedelta(1) start = datetime(chosendate.year, chosendate.month, chosendate.day)
start = datetime(chosendate.year, chosendate.month, chosendate.day) if len(args) > 1:
if len(args) > 1: stop = parse_input_date(args[1], periodicity)
stop = parse_input_date(args[1], periodicity) if stop is None:
if stop is None: stop = start
stop = start if start is None or stop is None:
if start is None or stop is None: sys.exit(1) # parse_input_date failed to parse date
sys.exit(1) # parse_input_date failed to parse date stop += ONEDAY # date_range() excludes stop boundary
stop += ONEDAY # date_range() excludes stop boundary stats_updater = StatsUpdater(cnx, start, stop)
stats_updater = StatsUpdater(session, cw_cnx, start, stop) stats_report = stats_updater.update_stats(self.config.skip_compress)
stats_report = stats_updater.update_stats(self.config.skip_compress) print '''=== Update Report ===
print '''=== Update Report ===
Number of periods imported : %(periods)s Number of periods imported : %(periods)s
Number of stat objects created : %(created)s Number of stat objects created : %(created)s
Number of stat objects updated : %(updated)s Number of stat objects updated : %(updated)s
...@@ -267,6 +236,6 @@ Number of stat objects already existed : %(exists no change)s ...@@ -267,6 +236,6 @@ Number of stat objects already existed : %(exists no change)s
Number of stat objects skipped : %(skipped)s Number of stat objects skipped : %(skipped)s
Number of stat objects ignored : %(ignored)s Number of stat objects ignored : %(ignored)s
Number of stat objects compressed : %(compressed)s Number of stat objects compressed : %(compressed)s
''' % stats_report ''' % stats_report
CWCTL.register(UpdateWebstatsCommand) CWCTL.register(UpdateWebstatsCommand)
...@@ -27,6 +27,7 @@ except ImportError: ...@@ -27,6 +27,7 @@ except ImportError:
_ = unicode _ = unicode
from cubicweb.req import FindEntityError from cubicweb.req import FindEntityError
from cubicweb.web import NotFound, Redirect from cubicweb.web import NotFound, Redirect
from cubicweb.devtools.fake import FakeRequest
from psycopg2 import DataError from psycopg2 import DataError
...@@ -136,32 +137,33 @@ def extract_stats_dict(filepath): ...@@ -136,32 +137,33 @@ def extract_stats_dict(filepath):
try: try:
stats_dict[section_name][key][SECTIONSPEC[section_name][index]] = value stats_dict[section_name][key][SECTIONSPEC[section_name][index]] = value
except IndexError: except IndexError:
pass pass
parsed_countdown -= 1 parsed_countdown -= 1
elif section_name and parsed_countdown == 0: elif section_name and parsed_countdown == 0:
section_name = None section_name = None
return stats_dict return stats_dict
def eid_from_url(req, value):
def eid_from_url(cnx, value):
''' return an eid from an url ''' ''' return an eid from an url '''
url_resolver = req.vreg['components'].select('urlpublisher', req = FakeRequest(vreg=cnx.vreg, url=value)
vreg=req.vreg) req.set_cnx(cnx)
req.url = lambda includeparams: value url_resolver = cnx.vreg['components'].select('urlpublisher',
req.relative_path = lambda includeparams: value[1:] vreg=cnx.vreg)
try: try:
pmid, rset = url_resolver.process(req, value) pmid, rset = url_resolver.process(req, value)
if rset and len(rset) == 1: if rset and len(rset) == 1:
return rset[0][0] return rset[0][0]
elif req.form.get('rql'): elif req.form.get('rql'):
rset = req.execute(req.form.get('rql')) rset = cnx.execute(req.form.get('rql'))
if rset and len(rset) == 1: if rset and len(rset) == 1:
return rset[0][0] return rset[0][0]
except (NotFound, DataError, Redirect): except (NotFound, DataError, Redirect):
pass pass
def get_or_create_statperiod(session, start, stop, stats_report={}): def get_or_create_statperiod(cnx, start, stop, stats_report={}):
rql = 'Any P WHERE P is StatPeriod, P start "%(start_date)s", P stop "%(end_date)s"' rql = 'Any P WHERE P is StatPeriod, P start "%(start_date)s", P stop "%(end_date)s"'
rset = session.execute(rql % rset = cnx.execute(rql %
{'start_date':start, {'start_date':start,
'end_date':stop}) 'end_date':stop})
if rset: if rset:
...@@ -169,11 +171,11 @@ def get_or_create_statperiod(session, start, stop, stats_report={}): ...@@ -169,11 +171,11 @@ def get_or_create_statperiod(session, start, stop, stats_report={}):
else: else:
stats_report.setdefault('periods', 0) stats_report.setdefault('periods', 0)
stats_report['periods'] += 1 stats_report['periods'] += 1
return session.create_entity('StatPeriod', start=start, stop=stop) return cnx.create_entity('StatPeriod', start=start, stop=stop)
def time_params(req): def time_params(cnx):
params = [] params = []
rset = req.execute('Any START ORDERBY START LIMIT 1 WHERE P is StatPeriod, P start START, P stop STOP HAVING STOP-START <= 2') rset = cnx.execute('Any START ORDERBY START LIMIT 1 WHERE P is StatPeriod, P start START, P stop STOP HAVING STOP-START <= 2')
for (item,) in rset: for (item,) in rset:
for first_day in date_range(previous_month(item), previous_month(datetime.now(), 5), incmonth=True): for first_day in date_range(previous_month(item), previous_month(datetime.now(), 5), incmonth=True):
delta = 2 delta = 2
...@@ -182,8 +184,8 @@ def time_params(req): ...@@ -182,8 +184,8 @@ def time_params(req):
return params return params
def compress_old_hits(req, update_stats={}, progressbar=True): def compress_old_hits(cnx, update_stats={}, progressbar=True):
tp = time_params(req) tp = time_params(cnx)
if progressbar: if progressbar:
pb = ProgressBar(len(tp), 55, title='Compressing old stats') pb = ProgressBar(len(tp), 55, title='Compressing old stats')
for start, stop, delta in tp: for start, stop, delta in tp:
...@@ -193,9 +195,9 @@ def compress_old_hits(req, update_stats={}, progressbar=True): ...@@ -193,9 +195,9 @@ def compress_old_hits(req, update_stats={}, progressbar=True):
'H period P, P start >= %(start)s, P stop <= %(stop)s, H stats_about E,'\ 'H period P, P start >= %(start)s, P stop <= %(stop)s, H stats_about E,'\
'P start START, P stop STOP HAVING STOP-START <= %(timedelta)s' 'P start START, P stop STOP HAVING STOP-START <= %(timedelta)s'
results = {} results = {}
type_rset = req.execute('DISTINCT Any C WHERE X is Hits, X hit_type C') type_rset = cnx.execute('DISTINCT Any C WHERE X is Hits, X hit_type C')
for (hit_type,) in type_rset: for (hit_type,) in type_rset:
results[hit_type] = req.execute(rql, results[hit_type] = cnx.execute(rql,
{'start': start, {'start': start,
'stop': stop, 'stop': stop,
'hit_type':hit_type, 'hit_type':hit_type,
...@@ -203,16 +205,16 @@ def compress_old_hits(req, update_stats={}, progressbar=True): ...@@ -203,16 +205,16 @@ def compress_old_hits(req, update_stats={}, progressbar=True):
if not any(results.values()): if not any(results.values()):
continue continue
# deleting statperiod deletes all associated hits # deleting statperiod deletes all associated hits
drset = req.execute('DELETE StatPeriod P WHERE P start >= %(start)s, P stop <= %(stop)s', drset = cnx.execute('DELETE StatPeriod P WHERE P start >= %(start)s, P stop <= %(stop)s',
{'start': start, {'start': start,
'stop': stop,}) 'stop': stop,})
update_stats['compressed'] += len(drset) update_stats['compressed'] += len(drset)
stp = get_or_create_statperiod(req, start, stop) stp = get_or_create_statperiod(cnx, start, stop)
for hit_type, rset in results.items(): for hit_type, rset in results.items():
for eid, count in rset: for eid, count in rset:
content_entity = req.entity_from_eid(eid) content_entity = cnx.entity_from_eid(eid)
# FIXME if Hits for period and content exist, update it ? # FIXME if Hits for period and content exist, update it ?
req.create_entity('Hits', hit_type=hit_type, period=stp, count=count, cnx.create_entity('Hits', hit_type=hit_type, period=stp, count=count,
stats_about=content_entity) stats_about=content_entity)
if progressbar: if progressbar:
pb.finish() pb.finish()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment