es.py

# -*- coding: utf-8 -*-
# copyright 2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr -- mailto:contact@logilab.fr
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import logging

from elasticsearch.exceptions import ConnectionError
from urllib3.exceptions import ProtocolError
from elasticsearch_dsl.connections import connections

from rql.utils import rqlvar_maker

INDEXABLE_TYPES = None


INDEX_SETTINGS = {'analysis':
                  {'analyzer':
                   {'default': {'filter': ['standard',
                                           'my_ascii_folding',
                                           'lowercase',
                                           'french_snowball'],
                                'tokenizer': 'standard'}},
                      'filter': {'my_ascii_folding': {'preserve_original': True,
                                                      'type': 'asciifolding'},
                                 'french_snowball': {'type': 'snowball',
                                                     'language': 'French'}
                                 }
                   },
                  }

# customization mechanism, in your cube, add your type as a key, and a list of
# additionnal attributes
# eg. CUSTOM_ATTRIBUTES['BlogEntry'] = ('description',)
CUSTOM_ATTRIBUTES = {}

log = logging.getLogger(__name__)


def indexable_types(schema):
    '''
    introspect indexable types
    '''
    global INDEXABLE_TYPES
    if INDEXABLE_TYPES is not None:
        return INDEXABLE_TYPES
    indexable_types = []
    skip_list = ('TrInfo', 'EmailAddress')
    for eschema in schema.entities():
        if eschema.type in skip_list:
            continue
        if not eschema.final:
            # check eschema.fulltext_relations() ? (skip wf_info_for ?
            # )
            if list(eschema.indexable_attributes()):
                indexable_types.append(eschema.type)
    INDEXABLE_TYPES = indexable_types
    return indexable_types


def fulltext_indexable_rql(etype, schema, eid=None):
    '''
    Generate RQL with fulltext_indexable attributes for a given entity type

    :eid:
       defaults to None, set it to an eid to get RQL for a single element (used in hooks)
    '''
    varmaker = rqlvar_maker()
    V = next(varmaker)
    rql = ['WHERE %s is %s' % (V, etype)]
    if eid:
        rql.append('%s eid %i' % (V, eid))
    var = next(varmaker)
    selected = []
    for rschema in schema.eschema(etype).indexable_attributes():
        attr = rschema.type
        var = next(varmaker)
        rql.append('%s %s %s' % (V, attr, var))
        selected.append((attr, var))
    for rschema, tschema in schema.eschema(etype).attribute_definitions():
        if tschema.type in ('Int', 'Float'):
            attr = rschema.type
            var = next(varmaker)
            rql.append('%s %s %s' % (V, attr, var))
            selected.append((attr, var))
    for attr in ('creation_date', 'modification_date',) + CUSTOM_ATTRIBUTES.get(etype, ()):
        var = next(varmaker)
        rql.append('%s %s %s' % (V, attr, var))
        selected.append(var)
    # TODO inlined relations ?
    return 'Any %s,%s %s' % (V, ','.join(selected),
                             ','.join(rql))


def get_connection(config):
    '''
    Get connection with config object, creates a persistent connexion and
    creates the initial index with custom settings
    '''
    try:
        return connections.get_connection()
    except KeyError:
        locations = config['elasticsearch-locations']
        index_name = config['index-name']
        if locations and index_name:
            # TODO sanitize locations
            es = connections.create_connection(hosts=locations.split(','),
                                               index=index_name,
                                               timeout=20)
            try:
                if not es.indices.exists(index=index_name):
                    # TODO could remove ignore=400 since checks does not exist
                    # ignore 400 caused by IndexAlreadyExistsException when creating an
                    es.indices.create(index=index_name,
                                      body=INDEX_SETTINGS,
                                      ignore=400)
            except (ConnectionError, ProtocolError):
                log.debug('Failed to index in hook, could not connect to ES')
            return es
        # TODO else ? raise KeyError - return None is OK?