Commit be43ca57 authored by Nicolas Chauvat's avatar Nicolas Chauvat
Browse files

remove statsd (closes #39)

parent 54bbd2f528c6
Pipeline #27957 passed with stages
in 8 minutes and 3 seconds
......@@ -842,13 +842,6 @@ class CubicWebConfiguration(CubicWebNoAppConfiguration):
'help': 'file where output logs should be written',
'group': 'main', 'level': 2,
}),
('statsd-endpoint',
{'type' : 'string',
'default': '',
'help': 'UDP address of the statsd endpoint; it must be formatted'
'like <ip>:<port>; disabled is unset.',
'group': 'main', 'level': 2,
}),
# email configuration
('smtp-host',
{'type' : 'string',
......@@ -1155,17 +1148,6 @@ the repository',
logconfig = join(self.apphome, 'logging.conf')
if exists(logconfig):
logging.config.fileConfig(logconfig)
# set the statsd address, if any
if self.get('statsd-endpoint'):
try:
address, port = self.get('statsd-endpoint').split(':')
port = int(port)
except:
self.error('statsd-endpoint: invalid address format ({}); '
'it should be "ip:port"'.format(self.get('statsd-endpoint')))
else:
import statsd_logger
statsd_logger.setup('cubicweb.%s' % self.appid, (address, port))
def available_languages(self, *args):
"""return available translation for an instance, by looking for
......
......@@ -41,7 +41,6 @@ from cubicweb.server.rqlannotation import RQLAnnotator, set_qdata
from cubicweb.server.ssplanner import (READ_ONLY_RTYPES, add_types_restriction,
prepare_plan)
from cubicweb.server.edition import EditedEntity
from cubicweb.statsd_logger import statsd_timeit, statsd_c
ETYPE_PYOBJ_MAP[Binary] = 'Bytes'
......@@ -491,7 +490,6 @@ class QuerierHelper(object):
return InsertPlan(self.schema, rqlst, args, cnx)
return ExecutionPlan(self.schema, rqlst, args, cnx)
@statsd_timeit
def execute(self, cnx, rql, args=None, build_descr=True):
"""execute a rql query, return resulting rows and their description in
a `ResultSet` object
......@@ -674,10 +672,8 @@ class RQLCache(object):
cachekey = _rql_cache_key(cnx, rql, args, eidkeys)
rqlst = self._cache[cachekey]
self.cache_hit += 1
statsd_c('cache_hit')
except KeyError:
self.cache_miss += 1
statsd_c('cache_miss')
rqlst = self._parse(rql)
# compute solutions for rqlst and return named args in query
# which are eids. Notice that if you may not need `eidkeys`, we
......
......@@ -53,7 +53,6 @@ from cubicweb.server.hook import CleanupDeletedEidsCacheOp
from cubicweb.server.edition import EditedEntity
from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results, rql2sql
from cubicweb.misc.source_highlight import highlight_terminal
from cubicweb.statsd_logger import statsd_timeit
ATTR_MAP = {}
......@@ -376,7 +375,6 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
if etype is not None:
cache.pop('Any X WHERE X eid %s, X is %s' % (eid, etype), None)
@statsd_timeit
def sqlexec(self, cnx, sql, args=None):
"""execute the query and return its result"""
return self.process_result(self.doexec(cnx, sql, args))
......@@ -467,7 +465,6 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
# ISource interface #######################################################
@statsd_timeit
def compile_rql(self, rql, sols):
rqlst = self.repo.vreg.rqlhelper.parse(rql)
rqlst.restricted_vars = ()
......@@ -489,7 +486,6 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
authentifier.set_schema(self.schema)
clear_cache(self, 'need_fti_indexation')
@statsd_timeit
def authenticate(self, cnx, login, **kwargs):
"""return CWUser eid for the given login and other authentication
information found in kwargs, else raise `AuthenticationError`
......@@ -682,7 +678,6 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
sql = self.sqlgen.delete('%s_relation' % rtype, attrs)
self.doexec(cnx, sql, attrs)
@statsd_timeit
def doexec(self, cnx, query, args=None, rollback=True, rql_query_tracing_token=None):
"""Execute a query.
it's a function just so that it shows up in profiling
......@@ -762,7 +757,6 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
emit_to_debug_channel("sql", query_debug_informations)
return cursor
@statsd_timeit
def doexecmany(self, cnx, query, args):
"""Execute a query.
it's a function just so that it shows up in profiling
......
# copyright 2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""Simple statsd_ logger for cubicweb.
This module is meant to be configured by setting a couple of global variables:
- ``bucket`` global variable will be used as statsd bucket in every
statsd_ UDP sent packet.
`- `address`` is a pair (IP, port) specifying the address of the
statsd_ server
There are 3 kinds of statds_ message::
- ``statsd_c(context, n)`` is a simple function to send statsd_
counter-type of messages like::
<bucket>.<context>:<n>|c\n
- ``statsd_g(context, value)`` to send statsd_ gauge-type of messages
like::
<bucket>.<context>:<n>|g\n
- ``statsd_t(context, ms)`` to send statsd_ time-type of messages
like::
<bucket>.<context>:<ms>|ms\n
There is also a decorator (``statsd_timeit``) that may be used to
measure and send to the statsd_ server the time passed in a function
or a method and the number of calls. It will send a message like::
<bucket>.<funcname>:<ms>|ms\n<bucket>.<funcname>:1|c\n
.. _statsd: https://github.com/etsy/statsd
"""
import time
import socket
from contextlib import contextmanager
_bucket = 'cubicweb'
_address = None
_socket = None
def setup(bucket, address):
"""Configure the statsd endpoint
:param bucket: the name of the statsd bucket that will be used to
build messages.
:param address: the UDP endpoint of the statsd server. Must a
couple (ip, port).
"""
global _bucket, _address, _socket
packed = None
for family in (socket.AF_INET6, socket.AF_INET):
try:
packed = socket.inet_pton(family, address[0])
break
except socket.error:
continue
if packed is None:
return
_bucket, _address = bucket, address
_socket = socket.socket(family, socket.SOCK_DGRAM)
def teardown():
"""Unconfigure the statsd endpoint
This is most likely only useful for unit tests"""
global _bucket, _address, _socket
_bucket = 'cubicweb'
_address = None
_socket = None
def statsd_c(context, n=1):
if _address is not None:
_socket.sendto('{0}.{1}:{2}|c\n'.format(_bucket, context, n).encode(),
_address)
def statsd_g(context, value):
if _address is not None:
_socket.sendto('{0}.{1}:{2}|g\n'.format(_bucket, context, value).encode(),
_address)
def statsd_t(context, value):
if _address is not None:
_socket.sendto('{0}.{1}:{2:.4f}|ms\n'.format(_bucket, context, value).encode(),
_address)
class statsd_timeit(object):
__slots__ = ('callable',)
def __init__(self, callableobj):
self.callable = callableobj
@property
def __doc__(self):
return self.callable.__doc__
@property
def __name__(self):
return self.callable.__name__
def __call__(self, *args, **kw):
if _address is None:
return self.callable(*args, **kw)
t0 = time.time()
try:
return self.callable(*args, **kw)
finally:
dt = 1000 * (time.time() - t0)
msg = '{0}.{1}:{2:.4f}|ms\n{0}.{1}:1|c\n'.format(
_bucket, self.__name__, dt).encode()
_socket.sendto(msg, _address)
def __get__(self, obj, objtype):
"""Support instance methods."""
if obj is None: # class method or some already wrapped method
return self
import functools
return functools.partial(self.__call__, obj)
@contextmanager
def statsd_timethis(ctxmsg):
if _address is not None:
t0 = time.time()
try:
yield
finally:
if _address is not None:
dt = 1000 * (time.time() - t0)
msg = '{0}.{1}:{2:.4f}|ms\n{0}.{1}:1|c\n'.format(
_bucket, ctxmsg, dt).encode()
_socket.sendto(msg, _address)
# -*- coding: utf-8 -*-
# copyright 2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""unit tests for module cubicweb.statsd_logger"""
import threading
import socket
import time
import re
import pytest
from unittest import TestCase
from cubicweb import statsd_logger as statsd
UDP_PORT = None
RUNNING = True
SOCK = socket.socket(socket.AF_INET,
socket.SOCK_DGRAM)
SOCK.settimeout(0.1)
STATSD = None
DATA = []
def statsd_rcv():
while RUNNING:
try:
data, addr = SOCK.recvfrom(1024)
if data:
rcv = [row.strip().decode() for row in data.splitlines()]
DATA.extend(rcv)
except socket.timeout:
pass
def setUpModule(*args):
global UDP_PORT, STATSD
SOCK.bind(('127.0.0.1', 0))
UDP_PORT = SOCK.getsockname()[1]
STATSD = threading.Thread(target=statsd_rcv)
STATSD.start()
statsd.setup('test', ('127.0.0.1', UDP_PORT))
def tearDownModule(*args):
global RUNNING
RUNNING = False
STATSD.join()
statsd.teardown()
class StatsdTC(TestCase):
def setUp(self):
super(StatsdTC, self).setUp()
DATA[:] = []
def check_received(self, value):
for i in range(10):
if value in DATA:
break
time.sleep(0.01)
else:
self.assertIn(value, DATA)
def check_received_ms(self, value):
value = re.compile(value.replace('?', r'\d'))
for i in range(10):
if [x for x in DATA if value.match(x)]:
break
time.sleep(0.01)
else:
self.assertTrue([x for x in DATA if value.match(x)], DATA)
def test_statsd_c(self):
statsd.statsd_c('context')
self.check_received('test.context:1|c')
statsd.statsd_c('context', 10)
self.check_received('test.context:10|c')
def test_statsd_g(self):
statsd.statsd_g('context', 42)
self.check_received('test.context:42|g')
statsd.statsd_g('context', 'Igorrr')
self.check_received('test.context:Igorrr|g')
def test_statsd_t(self):
statsd.statsd_t('context', 1)
self.check_received('test.context:1.0000|ms')
statsd.statsd_t('context', 10)
self.check_received('test.context:10.0000|ms')
statsd.statsd_t('context', 0.12344)
self.check_received('test.context:0.1234|ms')
statsd.statsd_t('context', 0.12345)
self.check_received('test.context:0.1235|ms')
def test_decorator(self):
@statsd.statsd_timeit
def measure_me_please():
"some nice function"
return 42
self.assertEqual(measure_me_please.__doc__,
"some nice function")
measure_me_please()
self.check_received_ms('test.measure_me_please:0.0???|ms')
self.check_received('test.measure_me_please:1|c')
@pytest.mark.xfail(reason="this test randomly fail due to race condition and we don't uses "
"statsd anymore")
def test_context_manager(self):
with statsd.statsd_timethis('cm'):
time.sleep(0.1)
self.check_received_ms('test.cm:100.????|ms')
self.check_received('test.cm:1|c')
if __name__ == '__main__':
from unittest import main
main()
......@@ -71,7 +71,6 @@ cubicweb/sobjects/services.py
cubicweb/sobjects/test/unittest_notification.py
cubicweb/sobjects/test/unittest_register_user.py
cubicweb/sobjects/textparsers.py
cubicweb/statsd_logger.py
cubicweb/test/data/migration/0.1.0_common.py
cubicweb/test/data/migration/0.1.0_repository.py
cubicweb/test/data_schemareader/schema.py
......@@ -87,7 +86,6 @@ cubicweb/test/unittest_rqlrewrite.py
cubicweb/test/unittest_rset.py
cubicweb/test/unittest_rtags.py
cubicweb/test/unittest_schema.py
cubicweb/test/unittest_statsd.py
cubicweb/test/unittest_toolsutils.py
cubicweb/test/unittest_wfutils.py
cubicweb/toolsutils.py
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment