Commit 869aef8b authored by Denis Laxalde's avatar Denis Laxalde
Browse files

Get rid of file dropping support for cubicweb < 3.21

Bump required version of cubicweb to 3.24 as I'm not quire sure the code
will work with a version between 3.21 and 3.24 and 3.24 is already quite
old itself.

Related to #17133718.
parent e350ceb1da4b
......@@ -14,7 +14,7 @@ description = '"SKOS implementation for cubicweb"'
web = '' % distname
__depends__ = {
'cubicweb': '>= 3.20.5',
'cubicweb': '>= 3.24.0',
__recommends__ = {
'rdflib': '>= 4.1',
......@@ -20,7 +20,7 @@ BuildArch: noarch
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
BuildRequires: %{python} %{python}-setuptools
Requires: cubicweb >= 3.20.5
Requires: cubicweb >= 3.24.0
Requires: python-rdflib >= 4.1
%{?el6:Requires: python-redland}
......@@ -11,7 +11,7 @@ X-Python-Version: >= 2.6
Package: cubicweb-skos
Architecture: all
cubicweb-common (>= 3.20.4),
cubicweb-common (>= 3.24.0),
......@@ -13,7 +13,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <>.
"""SKOS Import code specific to cubicweb >= 3.21"""
"""SKOS Import code"""
from itertools import imap
......@@ -21,7 +21,6 @@ from six import text_type
from cubicweb.dataimport.importer import (ExtEntity, ExtEntitiesImporter, RelationMapping,
cwuri2eid, use_extid_as_cwuri)
from cubicweb.dataimport.importer import HTMLImportLog # noqa
from cubes.skos.sobjects import ext_dump_relations
# copyright 2015-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact --
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <>.
"""SKOS Import code specific to cubicweb < 3.21"""
from itertools import imap
from six import text_type
from cubes.skos import ExtEntity
from cubes.skos.dataimport import ExtEntitiesImporter, cwuri2eid
from cubes.skos.dataimport import HTMLImportLog # noqa
from cubes.skos.sobjects import ext_dump_relations
def store_skos_extentities(cnx, store, entities, import_log,
source=None, raise_on_error=False, extid_as_cwuri=True):
"""Add SKOS external entities to the store. Don't commit/flush any data.
* `cnx`, RQL connection to the CubicWeb instance
* `store`, store to use for the import
* `entities`, iterable (usualy a generator) on external entities to import
* `import_log`, import log instance to give to the store
* `source`, optional source in which existing entities will be looked for
(default to the system source)
* `raise_on_error`, boolean flag telling if we should fail on error or simply log it (default
to `False`)
* `extid_as_cwuri`, boolean flag telling if we should use the external entities'extid as
`cwuri` attribute of imported entities (default to `True`)
# only consider the system source for schemes and labels
if source is None:
source_eid = cnx.repo.system_source.eid
source_eid = source.eid
extid2eid = cwuri2eid(cnx, ('ConceptScheme', 'Label'), source_eid=source_eid)
# though concepts and external URIs may come from any source
extid2eid.update(cwuri2eid(cnx, ('Concept', 'ExternalUri')))
# plug function that turn previously known external uris by newly inserted concepts
restore_relations = {}
# pylint: disable=dangerous-default-value
def externaluri_to_concept(extentity, cnx=cnx, extid2eid=extid2eid,
eid = extid2eid[extentity.extid]
except KeyError:
if extentity.etype == 'Concept' and cnx.entity_metas(eid)['type'] == 'ExternalUri':
# We have replaced the external uri by the new concept. As entities.extid column is
# unique, we've to drop the external uri before inserting the concept, so we:
# 1. record every relations from/to the external uri,
# 2. remove it,
# 3. insert the concept and
# 4. reinsert relations using the concept instead
# 1. record relations from/to the external uri
restore_relations[extentity.extid] = ext_dump_relations(cnx, extid2eid, extentity)
# 2. remove the external uri entity
cnx.execute('DELETE ExternalUri X WHERE X eid %(x)s', {'x': eid})
# 3. drop its extid from the mapping to trigger insertion of the concept by the
# importer
del extid2eid[extentity.extid]
# 4. will be done in SKOSExtEntitiesImporter
return extentity
entities = imap(externaluri_to_concept, entities)
# plug function to detect the concept scheme
concept_schemes = []
def record_scheme(extentity):
if extentity.etype == 'ConceptScheme':
return extentity
entities = imap(record_scheme, entities)
etypes_order_hint = ('ConceptScheme', 'Concept', 'Label')
importer = SKOSExtEntitiesImporter(cnx, store, import_log, source=source, extid2eid=extid2eid,
stats = importer.import_entities(entities, use_extid_as_cwuri=extid_as_cwuri)
return stats, concept_schemes
class SKOSExtEntitiesImporter(ExtEntitiesImporter):
"""Override ExtEntitiesImporter to handle creation of additionnal relations to newly created
concepts that replace a former external uri, and to create ExternalUri entities for URIs used in
exact_match / close_match relations which have no known entity in the repository yet.
def __init__(self, *args, **kwargs):
self.restore_relations = kwargs.pop('restore_relations')
super(SKOSExtEntitiesImporter, self).__init__(*args, **kwargs)
def create_entity(self, extentity):
entity = super(SKOSExtEntitiesImporter, self).create_entity(extentity)
# (4.) restore relations formerly from/to an equivalent external uri
relations = self.restore_relations.pop(extentity.extid)
except KeyError:
return entity
for subject_eid, rtype, object_eid in relations:
if subject_eid is None:
subject_eid = entity.eid
if object_eid is None:
object_eid = entity.eid, rtype, object_eid)
return entity
def create_deferred_relations(self, deferred):
# create missing targets for exact_match and close_match relations
for rtype in ('exact_match', 'close_match'):
relations = deferred.get(rtype, ())
for _, object_uri in relations:
if object_uri not in self.extid2eid:
extentity = ExtEntity('ExternalUri', object_uri,
assert self.create_entity(extentity).cw_etype == 'ExternalUri'
return super(SKOSExtEntitiesImporter, self).create_deferred_relations(deferred)
def existing_relations(self, rtype):
"""return a set of (subject, object) eids already related by `rtype`"""
if rtype in ('exact_match', 'close_match'):
rql = 'Any X,O WHERE X %s O' % rtype
return set(tuple(x) for x in self.cnx.execute(rql))
return super(SKOSExtEntitiesImporter, self).existing_relations(rtype)
......@@ -32,11 +32,15 @@ from itertools import imap
from logilab.common.deprecation import deprecated
from cubicweb import schema, dataimport
from cubicweb.dataimport.importer import (
from cubicweb.predicates import match_kwargs, match_user_groups
from cubicweb.server import Service
from cubicweb.server.sources import datafeed
from cubes.skos import (POST_321, LABELS_RDF_MAPPING, register_skos_rdf_input_mapping,
from cubes.skos import (LABELS_RDF_MAPPING, register_skos_rdf_input_mapping,
rdfio, lcsv, to_unicode)
......@@ -271,11 +275,7 @@ def import_skos_extentities(cnx, entities, import_log,
return res
if POST_321:
# use new CW 3.21 dataimport API
from .post321_import import HTMLImportLog, ExtEntity, store_skos_extentities
from .pre321_import import HTMLImportLog, ExtEntity, store_skos_extentities
from .post321_import import store_skos_extentities # noqa E402
@deprecated('[skos 0.9] use lcsv_extentities then import_skos_extentities instead')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment