stores.py 17.1 KB
Newer Older
1
# copyright 2003-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
"""
Stores are responsible to insert properly formatted entities and relations into the database. They
have the following API::

    >>> user_eid = store.prepare_insert_entity('CWUser', login=u'johndoe')
    >>> group_eid = store.prepare_insert_entity('CWUser', name=u'unknown')
24
    >>> store.prepare_insert_relation(user_eid, 'in_group', group_eid)
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
    >>> store.flush()
    >>> store.commit()
    >>> store.finish()

Some store **requires a flush** to copy data in the database, so if you want to have store
independant code you should explicitly call it. (There may be multiple flushes during the
process, or only one at the end if there is no memory issue). This is different from the
commit which validates the database transaction. At last, the `finish()` method should be called in
case the store requires additional work once everything is done.

* ``prepare_insert_entity(<entity type>, **kwargs) -> eid``: given an entity
  type, attributes and inlined relations, return the eid of the entity to be
  inserted, *with no guarantee that anything has been inserted in database*,

* ``prepare_update_entity(<entity type>, eid, **kwargs) -> None``: given an
  entity type and eid, promise for update given attributes and inlined
  relations *with no guarantee that anything has been inserted in database*,

* ``prepare_insert_relation(eid_from, rtype, eid_to) -> None``: indicate that a
  relation ``rtype`` should be added between entities with eids ``eid_from``
  and ``eid_to``. Similar to ``prepare_insert_entity()``, *there is no
  guarantee that the relation will be inserted in database*,

* ``flush() -> None``: flush any temporary data to database. May be called
  several times during an import,

* ``commit() -> None``: commit the database transaction,

* ``finish() -> None``: additional stuff to do after import is terminated.

55
.. autoclass:: cubicweb.dataimport.stores.NullStore
56
57
.. autoclass:: cubicweb.dataimport.stores.RQLObjectStore
.. autoclass:: cubicweb.dataimport.stores.NoHookRQLObjectStore
58
.. autoclass:: cubicweb.dataimport.stores.MetadataGenerator
59
60
61
62
"""
import inspect
from datetime import datetime
from copy import copy
63
from itertools import count
64

65
66
import pytz

67
from logilab.common.decorators import cached
68
from logilab.common.deprecation import class_deprecated
69
70
71
72
73

from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES
from cubicweb.server.edition import EditedEntity


74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
class NullStore(object):
    """Store that mainly describe the store API.

    It may be handy to test input data files or to measure time taken by steps above the store
    (e.g. data parsing, importer, etc.): simply give a :class:`NullStore` instance instead of the
    actual store.
    """

    def __init__(self):
        self._eid_gen = count()

    def prepare_insert_entity(self, *args, **kwargs):
        """Given an entity type, attributes and inlined relations, return the inserted entity's
        eid.
        """
        return next(self._eid_gen)

    def prepare_update_entity(self, etype, eid, **kwargs):
        """Given an entity type and eid, update the corresponding entity with specified attributes
        and inlined relations.
        """

    def prepare_insert_relation(self, eid_from, rtype, eid_to, **kwargs):
        """Insert into the database a  relation ``rtype`` between entities with eids ``eid_from``
        and ``eid_to``.
        """

    def flush(self):
        """Flush internal data structures."""

    def commit(self):
        """Commit the database transaction."""

    def finish(self):
        """Import is terminated, do necessary cleanup."""


class RQLObjectStore(NullStore):
112
113
114
    """Store that works by making RQL queries, hence with all the cubicweb's machinery activated.
    """

Denis Laxalde's avatar
Denis Laxalde committed
115
    def __init__(self, cnx):
116
        self._cnx = cnx
Denis Laxalde's avatar
Denis Laxalde committed
117
        self._commit = cnx.commit
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
        # XXX 3.21 deprecated attributes
        self.eids = {}
        self.types = {}

    def rql(self, *args):
        """Execute a RQL query. This is NOT part of the store API."""
        return self._cnx.execute(*args)

    def prepare_insert_entity(self, *args, **kwargs):
        entity = self._cnx.create_entity(*args, **kwargs)
        self.eids[entity.eid] = entity
        self.types.setdefault(args[0], []).append(entity.eid)
        return entity.eid

    def prepare_update_entity(self, etype, eid, **kwargs):
        entity = self._cnx.entity_from_eid(eid)
134
        assert entity.cw_etype == etype, 'Trying to update with wrong type %s' % etype
135
136
137
138
139
140
141
142
143
144
145
146
147
        # XXX some inlined relations may already exists
        entity.cw_set(**kwargs)

    def prepare_insert_relation(self, eid_from, rtype, eid_to, **kwargs):
        self.rql('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype,
                 {'x': int(eid_from), 'y': int(eid_to)})

    def commit(self):
        return self._commit()


class NoHookRQLObjectStore(RQLObjectStore):
    """Store that works by accessing low-level CubicWeb's source API, with all hooks deactivated. It
148
149
150
151
    may be given a metadata generator object to handle metadata which are usually handled by hooks.

    Arguments:
    - `cnx`, a connection to the repository
152
    - `metagen`, optional :class:`MetadataGenerator` instance
153
154
155
156
157
    """

    def __init__(self, cnx, metagen=None):
        super(NoHookRQLObjectStore, self).__init__(cnx)
        if metagen is None:
158
159
160
            metagen = MetadataGenerator(cnx)
        if isinstance(metagen, MetadataGenerator):
            metagen = _MetaGeneratorBWCompatWrapper(metagen)
161
        self.metagen = metagen
162
163
164
165
        self._system_source = cnx.repo.system_source
        self._rschema = cnx.repo.schema.rschema
        self._create_eid = self._system_source.create_eid
        self._add_relation = self._system_source.add_relation
166
167
168
169
170
171
172
173
174
175
176
        self._nb_inserted_entities = 0
        self._nb_inserted_types = 0
        self._nb_inserted_relations = 0
        # deactivate security
        cnx.read_security = False
        cnx.write_security = False

    def prepare_insert_entity(self, etype, **kwargs):
        """Given an entity type, attributes and inlined relations, returns the inserted entity's
        eid.
        """
177
        for k, v in kwargs.items():
178
179
180
181
182
183
184
185
            kwargs[k] = getattr(v, 'eid', v)
        entity, rels = self.metagen.base_etype_dicts(etype)
        # make a copy to keep cached entity pristine
        entity = copy(entity)
        entity.cw_edited = copy(entity.cw_edited)
        entity.cw_clear_relation_cache()
        entity.cw_edited.update(kwargs, skipsec=False)
        cnx = self._cnx
186
        entity.eid = self._create_eid(cnx)
187
188
        entity_source = self.metagen.init_entity(entity)
        self._system_source.add_info(cnx, entity, entity_source)
189
        self._system_source.add_entity(cnx, entity)
190
        kwargs = dict()
191
        if inspect.getfullargspec(self._add_relation).varkw:
192
            kwargs['subjtype'] = entity.cw_etype
193
        for rtype, targeteids in rels.items():
194
            # targeteids may be a single eid or a list of eids
195
            inlined = self._rschema(rtype).inlined
196
197
            try:
                for targeteid in targeteids:
198
199
                    self._add_relation(cnx, entity.eid, rtype, targeteid,
                                       inlined, **kwargs)
200
            except TypeError:
201
202
                self._add_relation(cnx, entity.eid, rtype, targeteids,
                                   inlined, **kwargs)
203
204
205
206
207
208
209
210
211
212
213
        self._nb_inserted_entities += 1
        return entity.eid

    # XXX: prepare_update_entity is inherited from RQLObjectStore, it should be reimplemented to
    # actually skip hooks as prepare_insert_entity

    def prepare_insert_relation(self, eid_from, rtype, eid_to, **kwargs):
        """Insert into the database a  relation ``rtype`` between entities with eids ``eid_from``
        and ``eid_to``.
        """
        assert not rtype.startswith('reverse_')
214
215
216
217
        rschema = self._rschema(rtype)
        self._add_relation(self._cnx, eid_from, rtype, eid_to, rschema.inlined)
        if rschema.symmetric:
            self._add_relation(self._cnx, eid_to, rtype, eid_from, rschema.inlined)
218
219
220
        self._nb_inserted_relations += 1


221
222
223
224
225
226
227
228
229
230
class MetadataGenerator(object):
    """Class responsible for generating standard metadata for imported entities. You may want to
    derive it to add application specific's metadata. This class (or a subclass) may either be
    given to a nohook or massive store.

    Parameters:
    * `cnx`: connection to the repository
    * `baseurl`: optional base URL to be used for `cwuri` generation - default to config['base-url']
    * `source`: optional source to be used as `cw_source` for imported entities
    """
231
232
233
234
    META_RELATIONS = frozenset(META_RTYPES
                               - VIRTUAL_RTYPES
                               - set(('eid', 'cwuri',
                                      'is', 'is_instance_of', 'cw_source')))
235

236
    def __init__(self, cnx, baseurl=None, source=None, meta_skipped=()):
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
        self._cnx = cnx
        if baseurl is None:
            config = cnx.vreg.config
            baseurl = config['base-url'] or config.default_base_url()
        if not baseurl[-1] == '/':
            baseurl += '/'
        self._baseurl = baseurl
        if source is None:
            source = cnx.repo.system_source
        self.source = source
        self._now = datetime.now(pytz.utc)
        # attributes/relations shared by all entities of the same type
        self._etype_attrs = []
        self._etype_rels = []
        # attributes/relations specific to each entity
        self._entity_attrs = ['cwuri']
        rschema = cnx.vreg.schema.rschema
254
255
        self.meta_relations = self.META_RELATIONS - set(meta_skipped)
        for rtype in self.meta_relations:
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
            # skip owned_by / created_by if user is the internal manager
            if cnx.user.eid == -1 and rtype in ('owned_by', 'created_by'):
                continue
            if rschema(rtype).final:
                self._etype_attrs.append(rtype)
            else:
                self._etype_rels.append(rtype)

    # etype is provided in the 3 methods below as proven useful to custom implementation but not
    # used by the default implementation

    def etype_attrs(self, etype):
        """Return the list of attributes to be set for all entities of the given type."""
        return self._etype_attrs[:]

    def etype_rels(self, etype):
        """Return the list of relations to be set for all entities of the given type."""
        return self._etype_rels[:]

    def entity_attrs(self, etype):
        """Return the list of attributes whose value is set per instance, not per type, for the
        given type.
        """
        return self._entity_attrs[:]

    @cached
    def base_etype_attrs(self, etype):
        """Return a dictionary of attributes to be set for all entities of the given type."""
        attrs = {}
        for attr in self.etype_attrs(etype):
            genfunc = self._generator(attr)
            if genfunc:
                attrs[attr] = genfunc(etype)
        return attrs

    @cached
    def base_etype_rels(self, etype):
        """Return a dictionary of relations to be set for all entities of the given type."""
        rels = {}
        for rel in self.etype_rels(etype):
            genfunc = self._generator(rel)
            if genfunc:
                rels[rel] = genfunc(etype)
        return rels

    def init_entity_attrs(self, etype, eid, attrs):
        """Insert into an entity attrs dictionary attributes whose value is set per instance, not per
        type.
        """
        for attr in self.entity_attrs(etype):
            if attr in attrs:
                # already set, skip this attribute
                continue
            genfunc = self._generator(attr)
            if genfunc:
                attrs[attr] = genfunc(etype, eid, attrs)

    def _generator(self, rtype):
        return getattr(self, 'gen_%s' % rtype, None)

    def gen_cwuri(self, etype, eid, attrs):
        assert self._baseurl, 'baseurl is None while generating cwuri'
        return u'%s%s' % (self._baseurl, eid)

    def gen_creation_date(self, etype):
        return self._now

    def gen_modification_date(self, etype):
        return self._now

    def gen_created_by(self, etype):
        return self._cnx.user.eid

    def gen_owned_by(self, etype):
        return self._cnx.user.eid


class _MetaGeneratorBWCompatWrapper(object):
    """Class wrapping a MetadataGenerator to adapt it to the MetaGenerator interface.
    """
    META_RELATIONS = (META_RTYPES
                      - VIRTUAL_RTYPES
                      - set(('eid', 'cwuri',
                             'is', 'is_instance_of', 'cw_source')))

    def __init__(self, mdgenerator):
        self._mdgen = mdgenerator

    @cached
    def base_etype_dicts(self, etype):
        cnx = self._mdgen._cnx
        entity = cnx.vreg['etypes'].etype_class(etype)(cnx)
        # entity are "surface" copied, avoid shared dict between copies
        del entity.cw_extra_kwargs
        entity.cw_edited = EditedEntity(entity)
        attrs = self._mdgen.base_etype_attrs(etype)
        entity.cw_edited.update(attrs, skipsec=False)
        rels = self._mdgen.base_etype_rels(etype)
        return entity, rels

    def init_entity(self, entity):
        attrs = dict(entity.cw_edited)
        self._mdgen.init_entity_attrs(entity.cw_etype, entity.eid, attrs)
        entity.cw_edited.update(attrs, skipsec=False)
360
        return self._mdgen.source
361
362


Denis Laxalde's avatar
Denis Laxalde committed
363
class MetaGenerator(object, metaclass=class_deprecated):
364
365
366
367
368
369
370
371
    """Class responsible for generating standard metadata for imported entities. You may want to
    derive it to add application specific's metadata.

    Parameters:
    * `cnx`: connection to the repository
    * `baseurl`: optional base URL to be used for `cwuri` generation - default to config['base-url']
    * `source`: optional source to be used as `cw_source` for imported entities
    """
372
373
    __deprecation_warning__ = '[3.23] this class is deprecated, use MetadataGenerator instead'

374
    META_RELATIONS = MetadataGenerator.META_RELATIONS
375
376
377
378
379
380
381
382
383
384
385
386

    def __init__(self, cnx, baseurl=None, source=None):
        self._cnx = cnx
        if baseurl is None:
            config = cnx.vreg.config
            baseurl = config['base-url'] or config.default_base_url()
        if not baseurl[-1] == '/':
            baseurl += '/'
        self.baseurl = baseurl
        if source is None:
            source = cnx.repo.system_source
        self.source = source
387
        self._now = datetime.now(pytz.utc)
388
389
390
391
392
        # attributes/relations shared by all entities of the same type
        self.etype_attrs = []
        self.etype_rels = []
        # attributes/relations specific to each entity
        self.entity_attrs = ['cwuri']
393
        rschema = cnx.vreg.schema.rschema
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
        for rtype in self.META_RELATIONS:
            # skip owned_by / created_by if user is the internal manager
            if cnx.user.eid == -1 and rtype in ('owned_by', 'created_by'):
                continue
            if rschema(rtype).final:
                self.etype_attrs.append(rtype)
            else:
                self.etype_rels.append(rtype)

    @cached
    def base_etype_dicts(self, etype):
        entity = self._cnx.vreg['etypes'].etype_class(etype)(self._cnx)
        # entity are "surface" copied, avoid shared dict between copies
        del entity.cw_extra_kwargs
        entity.cw_edited = EditedEntity(entity)
        for attr in self.etype_attrs:
            genfunc = self.generate(attr)
            if genfunc:
                entity.cw_edited.edited_attribute(attr, genfunc(entity))
        rels = {}
        for rel in self.etype_rels:
            genfunc = self.generate(rel)
            if genfunc:
                rels[rel] = genfunc(entity)
        return entity, rels

    def init_entity(self, entity):
        for attr in self.entity_attrs:
            if attr in entity.cw_edited:
                # already set, skip this attribute
                continue
            genfunc = self.generate(attr)
            if genfunc:
                entity.cw_edited.edited_attribute(attr, genfunc(entity))
428
        return self.source
429
430
431
432
433
434
435
436
437

    def generate(self, rtype):
        return getattr(self, 'gen_%s' % rtype, None)

    def gen_cwuri(self, entity):
        assert self.baseurl, 'baseurl is None while generating cwuri'
        return u'%s%s' % (self.baseurl, entity.eid)

    def gen_creation_date(self, entity):
438
        return self._now
439
440

    def gen_modification_date(self, entity):
441
        return self._now
442
443
444
445
446
447

    def gen_created_by(self, entity):
        return self._cnx.user.eid

    def gen_owned_by(self, entity):
        return self._cnx.user.eid