Commit 333de19e authored by Arthur Lutz's avatar Arthur Lutz
Browse files

misc changes

parent add22d0cad76
......@@ -5,6 +5,14 @@ cubicweb integrated awstats frontend
To establish a link with the entity you want to build stats for
you need to create a relationship Hits "stats_about" X.
.. sourcecode:: python
class missing_stats_about(RelationDefinition):
name = 'stats_about'
subject = 'Hits'
object = ('BlogEntry', 'MicroBlogEntry')
Tips
----
......
......@@ -27,6 +27,8 @@ def url_count_from_stats(stats_dict):
returns two dictionnaries with eid as key and sequence of values as value
one for normal navigation, the other for rdf navigation
'''
if 'SIDER' not in stats_dict.keys():
return {}, {}
visit_count_dict = {}
visit_count_rdf_dict = {}
for item in stats_dict['SIDER'].values():
......@@ -67,9 +69,6 @@ class UpdateWebstatsCommand(Command):
options = [
("skip-compress", {"action": 'store_true',
'help' : u'Skip the compression of old daily hits into month stats'}),
# ('dry-run', {'action': 'store_true',
# 'help': "do not actually insert data in the database",
# }),
]
def get_current_stats_period(self, session, chosendate):
""" return a statperiod for the current month, if it doesn't exist, create it """
......@@ -106,9 +105,17 @@ class UpdateWebstatsCommand(Command):
input_format = {'month':'%m/%Y',
'day': '%d/%m/%Y',
'hour': '%d/%m/%Y-%Hh'}[periodicity]
start = datetime.strptime(args[0], input_format)
try:
start = datetime.strptime(args[0], input_format)
except ValueError:
print 'Error : %s not a proper date' % args[0]
return
if len(args) > 1:
stop = datetime.strptime(args[1], input_format)
try:
stop = datetime.strptime(args[1], input_format)
except ValueError:
print 'Error : %s not a proper date' % args[1]
return
else:
start = stop = datetime.now()
if stop is None:
......@@ -122,7 +129,7 @@ class UpdateWebstatsCommand(Command):
}
pb = ProgressBar(((stop+timedelta(days=1))-start).days, 70, title='Import')
for chosendate in date_range(start, stop+timedelta(days=1)):
#self.update_stats_for_date(session, chosendate, update_stats)
self.update_stats_for_date(session, chosendate, update_stats)
pb.update()
pb.finish()
if not self.config.skip_compress:
......@@ -138,7 +145,6 @@ Number of stat objects compressed : %(compressed)s
def update_stats_for_date(self, session, chosendate, update_stats):
stats_period = self.get_current_stats_period(session, chosendate)
update_stats['periods'] += 1
periodicity = session.vreg.config.get('awstats-periodicity', 'day') #FIXME s/day/month/
dateformat_in_file = self.choose_dateformat(periodicity)
domain = session.vreg.config.get('awstats-domain', '')
......
// TODO - customize by adding yaxis : {tickFormatter : suffixFormatter}, (when flot will be customizable)
function suffixFormatter(val, axis) {
if (val > 1e12)
return (val / 1e12).toFixed(axis.tickDecimals) + " T";
else if (val > 1e9)
return (val / 1e9).toFixed(axis.tickDecimals) + " G";
else if (val > 1e6)
return (val / 1e6).toFixed(axis.tickDecimals) + " M";
else if (val > 1000)
return (val / 1000).toFixed(axis.tickDecimals) + " k";
else
return val.toFixed(axis.tickDecimals);
}
\ No newline at end of file
......@@ -15,3 +15,14 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""cubicweb-awstats entity's classes"""
from cubicweb.entities import AnyEntity
class StatPeriod(AnyEntity):
__regid__ = 'StatPeriod'
def dc_title(self):
_ = self._cw._
return u'%s : %s - %s' % (_('Stat period'),
self.printable_value('start'),
self.printable_value('stop'))
......@@ -16,7 +16,7 @@
"""cubicweb-awstats schema"""
from yams.buildobjs import (EntityType, String, Int, Date, Datetime, Boolean,
from yams.buildobjs import (EntityType, String, Int, BigInt, Date, Datetime, Boolean,
SubjectRelation, RelationDefinition, RelationType)
_ = unicode
......@@ -30,15 +30,24 @@ MANAGER_PERMISSIONS = {
class StatPeriod(EntityType):
__permissions__ = MANAGER_PERMISSIONS
# FIXME - use DateTime if periodicity is hour ? (optimisation in db?)
start = Date()
stop = Date()
start = Date(indexed=True)
stop = Date(indexed=True)
class Hits(EntityType):
__permissions__ = MANAGER_PERMISSIONS
name = String(maxsize=128)
hit_type = String(maxsize=128)
count = Int()
period = SubjectRelation('StatPeriod', cardinality='?*')
hit_type = String(maxsize=128, indexed=True)
count = BigInt()
class period(RelationType):
subject = 'Hits'
object = 'StatPeriod'
cardinality = '1*'
composite = 'object'
inlined = True
class stats_about(RelationType):
inlined = True
cardinality = '?*'
......@@ -158,6 +158,10 @@ def get_or_create_statperiod(session, start, stop):
def compress_old_hits(req, update_stats={}):
# TODO roll complete 12 months into a year
rql = 'Any X WHERE X is StatPeriod, X start START, X stop STOP, X stop <= %(date)s HAVING STOP-START <= 20'
rset = req.execute(rql, {'date':previous_month(datetime.now(), 4)})
if not rset:
return
pb = ProgressBar(16, 55, title='Compressing old stats')
for monthsbefore in range(4,20):
pb.update()
......@@ -181,10 +185,12 @@ def compress_old_hits(req, update_stats={}):
created_entities = []
for hit_type, rset in results.items():
for eid, count in rset:
print 'creating', hit_type, start, stop, count, eid
content_entity = req.entity_from_eid(eid)
created_entities.append(req.create_entity('Hits', hit_type=hit_type, period=stp, count=count,
stats_about=content_entity))
try:
created_entities.append(req.create_entity('Hits', hit_type=hit_type, period=stp, count=count,
stats_about=content_entity))
except Exception:
pass
delete_rql = 'DISTINCT Hits X WHERE X period P, P start >= %(start)s, P stop <= %(stop)s, S start START, S stop STOP HAVING STOP-START <= 20'
rset = req.execute(delete_rql, {'start': start,
'stop': stop})
......
......@@ -16,6 +16,7 @@
"""cubicweb-awstats views/forms/actions/components for web ui"""
from cubicweb.utils import RepeatList
from cubicweb.view import EntityView
from cubicweb.web.views import primary, navigation
from cubicweb.selectors import is_instance
......@@ -33,17 +34,21 @@ class StatPeriodPrimaryView(primary.PrimaryView):
column_types_aggr = None #tuple of tuples of types
def cell_call(self, row, col):
_ = self._cw._
req = self._cw
self.w(u'<div id="statperiod"/>')
self.w(u'<div id="statperiod">')
entity = self.cw_rset.get_entity(row, col)
self.w(u'<h1>%s %s - %s</h1>' % (_('Statistics for period :'), entity.start, entity.stop) )
rset = self._cw.execute('Any C GROUPBY C WHERE X is Hits, X hit_type C')
self.w(u'<h1>%s %s - %s</h1>' % (_('Statistics for period :'),
entity.printable_value('start'),
entity.printable_value('stop')) )
# TODO - display length
rset = self._cw.execute('DISTINCT Any C WHERE X is Hits, X hit_type C')
self.w(u'<a href="%s">%s</a>' % (entity.absolute_url(showall=1),
_('show all results')))
for index, hit_type in enumerate(rset):
self.w(u'<h3>%s</h3>' % hit_type[0])
for (hit_type,) in rset:
self.w(u'<h3>%s</h3>' % hit_type)
rql = 'Any X, C ORDERBY C DESC %(limit)s WHERE H stats_about X, H hit_type "%(type)s",'\
'H count C, H period P, P eid %%(e)s' % {'type': hit_type[0],
'H count C, H period P, P eid %%(e)s' % {'type': hit_type,
'limit': req.form.get('showall') and ' ' or 'LIMIT 20' }
if self.column_types_aggr:
self.w(u'<table><tr>')
......@@ -51,12 +56,20 @@ class StatPeriodPrimaryView(primary.PrimaryView):
self.w(u'<td>')
typedrql = rql + ', X is in (%s)' % ','.join(types)
rset = self._cw.execute(typedrql, {'e':entity.eid})
self.w(self._cw.view('table', rset, 'null'))
self.wview('table', rset, 'null')
# cf rql/editextensions.py unset_limit
#refactor below
nolimit_rql = typedrql.replace('LIMIT 20', '')
self.w(u'<a href="%s">Export CSV</a>' % self._cw.build_url('', rql=nolimit_rql % {'e':entity.eid},
vid='csvexport'))
self.w(u'</td>')
self.w(u'</tr></table>')
else:
rset = self._cw.execute(rql, {'e':entity.eid})
self.w(self._cw.view('table', rset, 'null'))
self.wview('table', rset, 'null')
nolimit_rql = rql.replace('LIMIT 20', '')
self.w(u'<a href="%s">Export CSV</a>' % self._cw.build_url('', rql=nolimit_rql % {'e':entity.eid},
vid='csvexport'))
self.w(u'</div>')
......@@ -88,13 +101,16 @@ class StatGraph(EntityView):
entity = self.cw_rset.get_entity(row, col)
self.w(u'<h1>%s %s</h1>' % (_('Graph of hits for'),
entity.dc_title()))
for constraint in (' HAVING O-S <= 20',
' HAVING O-S >= 20'):
rql = 'Any S, HITS WHERE H hit_type "normal", H count HITS, H stats_about E, H period P, P start S, P stop O, E eid %(e)s'
rset = self._cw.execute(rql + constraint, {'e':entity.eid})
self.w(u'%s' % (rql % {'e':entity.eid}))
rql = ('DISTINCT Any S, HITS ORDERBY S WITH S, HITS BEING ('
'(Any START, HITS WHERE H count HITS, H period P, P start START, P stop STOP, H stats_about E, E eid %%(e)s %s)'
' UNION '
'(Any START, 0 WHERE P is StatPeriod, P start START, P stop STOP, NOT EXISTS(H period P, H stats_about E, E eid %%(e)s) %s))')
for constraint in (' HAVING STOP-START <= 20',
' HAVING STOP-START >= 20'):
rset = self._cw.execute(rql % (constraint, constraint), {'e':entity.eid})
self.w(self._cw.view('plot', rset, 'null'))
class StatPeriodChart(EntityView):
__regid__ = 'periodchart'
__select__ = is_instance('StatPeriod')
......
......@@ -28,7 +28,7 @@ from logilab.common.textutils import BYTE_UNITS
from cubicweb.view import StartupView
from cubicweb.web.views import forms
from cubicweb.web.formfields import StringField, DateField
from cubicweb.web import formwidgets as fwdgs
from cubicweb.web import formwidgets as fwdgs, httpcache
from cubes.awstats.utils import SECTIONSPEC, SECTIONLABELS, \
extract_stats_dict, ORIGIN_LABELS
......@@ -93,6 +93,7 @@ def convert_to_bytes(value):
class AwstatsRefreshForm(forms.FieldsForm):
"""Form to filter and select what stats are being displayed"""
__regid__ = 'select-awstats'
#FIXME def action cf below
action = '/?vid=awstats'
domain = StringField(widget=fwdgs.Select(attrs={'onchange':'this.form.submit()'}),
label=_('Domain:'),
......@@ -203,18 +204,21 @@ class AwstatsView(StartupView):
class WebStatsRefreshForm(forms.FieldsForm):
"""Form to filter and select what stats are being displayed"""
__regid__ = 'select-webstats'
action = '/?vid=webstats'
start = DateField(label=_('Start:'),)
stop = DateField(label=_('Stop:'),)
limit = StringField(label=_('Number of results :'),
choices=[u'%s' % i for i in (10,25,50,100,200,500)])
form_buttons = [fwdgs.SubmitButton(label=_('Apply'))]
@property
def action(self):
return self._cw.build_url('', vid='webstats')
class StatPeriodsView(StartupView):
""" Web stats view - build from StatPeriods and Hits in cubicweb """
__regid__ = 'webstats'
column_types = None
http_cache_manager = httpcache.NoHTTPCacheManager
def call(self):
req = self._cw
......@@ -236,7 +240,10 @@ class StatPeriodsView(StartupView):
'stop': stop,
'duration':duration.days}))
self.description()
rset = self._cw.execute('Any C GROUPBY C WHERE X is Hits, X hit_type C')
# TODO - plus gros
self.w(u'<a href="%s">%s</a>' % (self._cw.build_url(rql='Any X ORDERBY S WHERE X is StatPeriod, X start S, X stop E HAVING E-S >= 20'),
_('Navigate previous statistics by month')))
rset = self._cw.execute('DISTINCT Any T WHERE X is Hits, X hit_type T')
for index, hit_type in enumerate(rset):
self.w(u'<h3>%s</h3>' % hit_type[0])
rql = 'Any X, SUM(C) GROUPBY X ORDERBY 2 DESC %(limit)s WHERE H stats_about X, ' \
......@@ -247,10 +254,10 @@ class StatPeriodsView(StartupView):
'stop': stop,
}
if self.column_types:
self.w(u'<table><tr>')
self.w(u'<table class="webstats"><tr>')
for etype in self.column_types:
self.w(u'<td>')
typedrql = rql + ', X is %s' % etype
self.w(u'<td class="webstats">')
typedrql = rql + ', X is in (%s)' % ','.join(etype)
rset = self._cw.execute(typedrql)
self.generate_table_form(rset, etype)
nolimit_rql = typedrql.replace('LIMIT %s' % limit, '')
......@@ -261,6 +268,9 @@ class StatPeriodsView(StartupView):
else:
rset = self._cw.execute(rql)
self.generate_table_form(rset)
nolimit_rql = rql.replace('LIMIT %s' % limit, '')
self.w(u'<a href="%s">Export CSV</a>' % self._cw.build_url(rql=nolimit_rql,
vid='csvexport'))
self.w(u'</div>')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment