# HG changeset patch # User Arthur Lutz <arthur.lutz@logilab.fr> # Date 1465391900 -7200 # Wed Jun 08 15:18:20 2016 +0200 # Node ID cc6df274a4ca975f55875e6618dd723fba8a002f # Parent f69eb7eca7c06b9bbf027641857d60fddfc77623 [ccplugin] dry-run for bulk import too diff --git a/ccplugin.py b/ccplugin.py --- a/ccplugin.py +++ b/ccplugin.py @@ -25,16 +25,17 @@ # TODO optimisation : json serialize on one side, send to ES on the other # TODO progress bar -def bulk_actions(rset, index_name, etype): +def bulk_actions(rset, index_name, etype, dry_run=False): for entity in rset.entities(): serializer = entity.cw_adapt_to('ISerializable') json = serializer.serialize() - yield {'_op_type': 'index', - '_index': index_name, - '_type': etype, - '_id': entity.eid, - '_source': json - } + if not dry_run: + yield {'_op_type': 'index', + '_index': index_name, + '_type': etype, + '_id': entity.eid, + '_source': json + } class IndexInES(Command): @@ -82,12 +83,13 @@ if self.config.debug: print(u'indexing {} {}'.format(etype, len(rset))) if self.config.bulk: - # success, failed = bulk(es, bulk_actions(rset, index_name, etype)) - # if self.config.debug: - # print(u'ES bulk : {} success {} failed'.format(success, failed)) - list(parallel_bulk(es, bulk_actions(rset, index_name, etype), - raise_on_error=False, - raise_on_exception=False)) + for x in parallel_bulk(es, bulk_actions(rset, + index_name, + etype, + dry_run=self.config.dry_run), + raise_on_error=False, + raise_on_exception=False): + pass else: for entity in rset.entities(): # TODO add specific IFTIES adapter