# HG changeset patch
# User Arthur Lutz <arthur.lutz@logilab.fr>
# Date 1463586684 -7200
# Wed May 18 17:51:24 2016 +0200
# Node ID aea1ea565d4ef4f8a510a788e57a4d1b7b20f791
# Parent 21b992410538ea8da81ce788fbb410ca6166e392
skip indexing when locations and index-name are not configured
diff --git a/ccplugin.py b/ccplugin.py
--- a/ccplugin.py
+++ b/ccplugin.py
@@ -45,38 +45,41 @@
config = cnx.vreg.config
schema = cnx.vreg.schema
locations = config['elasticsearch-locations']
- index_name = config.get('index-name') or 'cubicweb'
- es = Elasticsearch(locations and locations.split(',') or None)
- if not self.config.dry_run:
- # ignore 400 caused by IndexAlreadyExistsException when creating an
- # index
- es.indices.create(
- index=index_name, body=index_settings(), ignore=400)
+ index_name = config['index-name']
+ if locations and index_name:
+ es = Elasticsearch(locations)
+ if not self.config.dry_run:
+ # ignore 400 caused by IndexAlreadyExistsException when creating an
+ # index
+ es.indices.create(
+ index=index_name, body=index_settings(), ignore=400)
- if self.config.debug:
- print(u'found indexable_types {}'.format(
- ','.join(indexable_types(schema))))
- for etype in indexable_types(schema):
- rset = cnx.execute(
- 'Any X WHERE X is %(etype)s' % {'etype': etype})
if self.config.debug:
- print(u'indexing {} {}'.format(etype, len(rset)))
- for entity in rset.entities():
- # TODO add specific IFTIES adapter
- serializer = entity.cw_adapt_to('ISerializable')
- json = serializer.serialize()
- # TODO remove non indexable data or (better) serialize only
- if not self.config.dry_run:
- es.index(index=index_name, doc_type=etype, body=json)
- # TODO optimize with elasticsearch.helpers.bulk
- # or elasticsearch.helpers.parallel_bulk
- # or elasticsearch.helpers.streaming_bulk
- # TODO optimisation : json serialize on one side, send to ES on the other
- # TODO progress bar
+ print(u'found indexable_types {}'.format(
+ ','.join(indexable_types(schema))))
+ for etype in indexable_types(schema):
+ rset = cnx.execute(
+ 'Any X WHERE X is %(etype)s' % {'etype': etype})
if self.config.debug:
- print(u'.', end=u'')
+ print(u'indexing {} {}'.format(etype, len(rset)))
+ for entity in rset.entities():
+ # TODO add specific IFTIES adapter
+ serializer = entity.cw_adapt_to('ISerializable')
+ json = serializer.serialize()
+ # TODO remove non indexable data or (better) serialize only
+ if not self.config.dry_run:
+ es.index(index=index_name, doc_type=etype, body=json)
+ # TODO optimize with elasticsearch.helpers.bulk
+ # or elasticsearch.helpers.parallel_bulk
+ # or elasticsearch.helpers.streaming_bulk
+ # TODO optimisation : json serialize on one side, send to ES on the other
+ # TODO progress bar
+ if self.config.debug:
+ print(u'.', end=u'')
+ if self.config.debug:
+ print(u'')
+ else:
if self.config.debug:
- print(u'')
-
+ print(u'no elasticsearch configuration found, skipping')
CWCTL.register(IndexInES)
diff --git a/hooks.py b/hooks.py
--- a/hooks.py
+++ b/hooks.py
@@ -46,13 +46,14 @@
def __call__(self):
locations = self._cw.vreg.config['elasticsearch-locations']
index_name = self._cw.vreg.config['index-name']
- serializer = self.entity.cw_adapt_to('ISerializable')
- json = serializer.serialize()
- es = Elasticsearch(locations and locations.split(',') or None)
- try:
- # TODO option pour coté async ?
- es.index(index=index_name,
- doc_type=self.entity.cw_etype,
- body=json)
- except (ConnectionError, ProtocolError):
- log.debug('Failed to index in hook, could not connect to ES')
+ if locations and index_name:
+ serializer = self.entity.cw_adapt_to('ISerializable')
+ json = serializer.serialize()
+ es = Elasticsearch(locations and locations.split(','))
+ try:
+ # TODO option pour coté async ?
+ es.index(index=index_name,
+ doc_type=self.entity.cw_etype,
+ body=json)
+ except (ConnectionError, ProtocolError):
+ log.debug('Failed to index in hook, could not connect to ES')
diff --git a/site_cubicweb.py b/site_cubicweb.py
--- a/site_cubicweb.py
+++ b/site_cubicweb.py
@@ -3,15 +3,17 @@
{'type': 'string',
'default': '',
'help': 'Elastic Search location (eg. 192.168.0.23:9200), '
- 'this can be a list of locations (192.168.0.23:9200,192.168.0.24:9200,'
- ' you can also include the scheme (eg. http://192.168.0.23:9200) ',
+ 'this can be a list of locations (192.168.0.23:9200,192.168.0.24:9200, '
+ 'you can also include the scheme (eg. http://192.168.0.23:9200) '
+ 'warning: if this is not defined indexing will be disabled (no localhost default)',
'group': 'elasticsearch',
'level': 5,
}),
('index-name',
{'type': 'string',
- 'default': 'cubicweb',
- 'help': 'Elastic Search index name (eg. cubicweb)',
+ 'default': '',
+ 'help': 'Elastic Search index name (eg. cubicweb)'
+ 'warning: if this is not defined indexing will be disabled (no index name default)',
'group': 'elasticsearch',
'level': 5,
}),
diff --git a/test/test_elastic_search.py b/test/test_elastic_search.py
--- a/test/test_elastic_search.py
+++ b/test/test_elastic_search.py
@@ -25,6 +25,8 @@
self.orig_config_for = CubicWebConfiguration.config_for
config_for = lambda appid: self.config # noqa
CubicWebConfiguration.config_for = staticmethod(config_for)
+ self.config['elasticsearch-locations'] = 'http://10.1.1.1:9200'
+ self.config['index-name'] = 'unittest_index_name'
def to_test_etypes(self):
with self.admin_access.repo_cnx() as cnx:
@@ -79,7 +81,7 @@
self.assert_(cnx.execute('Any X WHERE X is %(etype)s' %
{'etype': indexable_types(cnx.repo)[0]}))
create.assert_called_with(
- ignore=400, index='cubicweb', body=index_settings())
+ ignore=400, index='unittest_index_name', body=index_settings())
index.assert_called()
# TODO ? check called data