Commit e918b981 authored by Arthur Lutz's avatar Arthur Lutz

use cnx to database

parent fc379159c6b1
......@@ -23,7 +23,7 @@ from utils import SECTIONSPEC, extract_stats_dict, eid_from_url, \
get_or_create_statperiod, compress_old_hits
def url_count_from_stats(session, stats_dict):
def url_count_from_stats(cnx, stats_dict):
'''
parse most visited urls in stats_dict generated from awstats txt file
......@@ -37,7 +37,8 @@ def url_count_from_stats(session, stats_dict):
for item in stats_dict['SIDER'].values():
url = item[SECTIONSPEC['SIDER'][0]]
hits = int(item[SECTIONSPEC['SIDER'][1]])
eid = eid_from_url(session, url)
req = cnx.request()
eid = eid_from_url(req, url)
if not eid:
continue
if 'rdf' in url:
......@@ -72,8 +73,9 @@ def track_progress(iterable, nb_ops=None, pb_size=20, pb_title=''):
class StatsUpdater(object):
def __init__(self, session, start, stop):
def __init__(self, session, cnx, start, stop):
self.session = session
self.cnx = cnx
self.config = session.vreg.config
self.start = start
self.stop = stop
......@@ -138,12 +140,13 @@ class StatsUpdater(object):
if not skip_compress:
compress_old_hits(self.session, stats_report)
self.session.commit()
self.session.set_cnxset()
return stats_report
def _update_stats_for_date(self, chosendate, stats_report):
stats_dict = extract_stats_dict(self.awstats_filepath(chosendate))
stats_period = self.stats_period_for_date(chosendate, stats_report)
normal_dict, rdf_dict = url_count_from_stats(self.session, stats_dict)
normal_dict, rdf_dict = url_count_from_stats(self.cnx, stats_dict)
for count_dict, hit_type in ((normal_dict, u'normal'),
(rdf_dict, u'rdf')):
for eid, values in count_dict.items():
......@@ -152,6 +155,8 @@ class StatsUpdater(object):
stats_report[status] += 1
def _update_hits_for_eid(self, eid, values, stats_period, hit_type):
self.session.commit()
self.session.set_cnxset()
visit_count = visit_count_rdf = 0
total_hits = sum([item[0] for item in values])
try:
......@@ -164,7 +169,8 @@ class StatsUpdater(object):
hit = self.all_hits[(eid, stats_period.eid, hit_type)]
except KeyError: # no hit yet, create one
status = 'created'
hit = self.session.create_entity('Hits', count=total_hits, hit_type=hit_type,
req = self.cnx.request()
hit = req.create_entity('Hits', count=total_hits, hit_type=hit_type,
period=stats_period, stats_about=entity)
# append it to the cache
self.all_hits[(eid, stats_period.eid, hit_type)] = hit
......@@ -224,6 +230,7 @@ class UpdateWebstatsCommand(Command):
break
session = repo._get_session(cnx.sessionid)
# XXX keep reference on cnx otherwise cnx.__del__ will cause trouble
cnx.use_web_compatible_requests(session.vreg.config['base-url'])
return cnx, session
def run(self, args):
......@@ -248,7 +255,7 @@ class UpdateWebstatsCommand(Command):
if start is None or stop is None:
sys.exit(1) # parse_input_date failed to parse date
stop += ONEDAY # date_range() excludes stop boundary
stats_updater = StatsUpdater(session, start, stop)
stats_updater = StatsUpdater(session, cw_cnx, start, stop)
stats_report = stats_updater.update_stats(self.config.skip_compress)
print '''=== Update Report ===
Number of periods imported : %(periods)s
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment