Commit 1b3bab5b authored by Sylvain Thénault's avatar Sylvain Thénault
Browse files

[ms, entity metas] add 'actual source' to entities table / base entity...

[ms, entity metas] add 'actual source' to entities table / base entity metadata cache. Closes #1767090

this is needed since for entities from 'copy based sources' such as
datafeed, we want entity.cw_metainformation() to return as 'source'
the datafeed source, not the system source (ie the source where the
entity is actually stored).

For both performance and bootstraping reasons, we should store this
information in the `entities` table and in the _type_source cache.
parent 86e9632a4e9c
......@@ -22,7 +22,7 @@ software
modname = distname = "cubicweb"
numversion = (3, 13, 0)
numversion = (3, 13, 1)
version = '.'.join(str(num) for num in numversion)
description = "a repository of entities / relations for knowledge management"
......
......@@ -346,9 +346,9 @@ class DBAPIRequest(RequestSessionBase):
# server session compat layer #############################################
def describe(self, eid):
def describe(self, eid, asdict=False):
"""return a tuple (type, sourceuri, extid) for the entity with id <eid>"""
return self.cnx.describe(eid)
return self.cnx.describe(eid, asdict)
def source_defs(self):
"""return the definition of sources used by the repository."""
......@@ -674,8 +674,16 @@ class Connection(object):
return self._repo.get_option_value(option, foreid)
@check_not_closed
def describe(self, eid):
return self._repo.describe(self.sessionid, eid, **self._txid())
def describe(self, eid, asdict=False):
metas = self._repo.describe(self.sessionid, eid, **self._txid())
if asdict:
if len(metas) == 3:
d = dict(zip(('type', 'source', 'extid'), metas))
d['asource'] = d['source']
return d
return dict(zip(('type', 'source', 'extid', 'asource'), metas))
# XXX :-1 for cw compat, use asdict=True for full information
return metas[:-1]
# db-api like interface ####################################################
......
......@@ -395,8 +395,10 @@ class Entity(AppObject):
@cached
def cw_metainformation(self):
res = dict(zip(('type', 'source', 'extid'), self._cw.describe(self.eid)))
res['source'] = self._cw.source_defs()[res['source']]
res = self._cw.describe(self.eid, asdict=True)
# use 'asource' and not 'source' since this is the actual source,
# while 'source' is the physical source (where it's stored)
res['source'] = self._cw.source_defs()[res.pop('asource')]
return res
def cw_check_perm(self, action):
......
......@@ -154,7 +154,7 @@ class ChangeEntityUpdateCaches(hook.Operation):
entity = self.entity
extid = entity.cw_metainformation()['extid']
repo._type_source_cache[entity.eid] = (
entity.__regid__, self.newsource.uri, None)
entity.__regid__, self.newsource.uri, None, self.newsource.uri)
if self.oldsource.copy_based_source:
uri = 'system'
else:
......@@ -177,6 +177,7 @@ class ChangeEntitySourceDeleteHook(MetaDataHook):
schange = self._cw.transaction_data.setdefault('cw_source_change', {})
schange[self.eidfrom] = self.eidto
class ChangeEntitySourceAddHook(MetaDataHook):
__regid__ = 'cw.metadata.source-change'
__select__ = MetaDataHook.__select__ & hook.match_rtype('cw_source')
......@@ -204,10 +205,12 @@ class ChangeEntitySourceAddHook(MetaDataHook):
# source='system'. External source will then have consider case
# where `extid2eid` return a negative eid as 'this entity was known
# but has been moved, ignore it'.
self._cw.system_sql('UPDATE entities SET eid=-eid,source=%(source)s WHERE eid=%(eid)s',
self._cw.system_sql('UPDATE entities SET eid=-eid,source=%(source)s '
'WHERE eid=%(eid)s',
{'eid': self.eidfrom, 'source': newsource.name})
attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': None,
'source': 'system', 'mtime': datetime.now()}
'source': 'system', 'asource': 'system',
'mtime': datetime.now()}
self._cw.system_sql(syssource.sqlgen.insert('entities', attrs), attrs)
# register an operation to update repository/sources caches
ChangeEntityUpdateCaches(self._cw, entity=entity,
......
# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
......@@ -35,6 +35,12 @@ def _add_relation_definition_no_perms(subjtype, rtype, objtype):
ss.execschemarql(rql, rdef, ss.rdef2rql(rdef, CSTRMAP, groupmap=None))
commit(ask_confirm=False)
if applcubicwebversion == (3, 13, 0) and cubicwebversion >= (3, 13, 1):
sql('ALTER TABLE entities ADD COLUMN asource VARCHAR(64)')
sql('INSERT INTO entities(asource) '
'SELECT cw_name FROM cw_CWSource, cw_source_relation '
'WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid')
if applcubicwebversion == (3, 6, 0) and cubicwebversion >= (3, 6, 0):
CSTRMAP = dict(rql('Any T, X WHERE X is CWConstraintType, X name T',
ask_confirm=False))
......
......@@ -409,7 +409,7 @@ class RequestSessionBase(object):
# abstract methods to override according to the web front-end #############
def describe(self, eid):
def describe(self, eid, asdict=False):
"""return a tuple (type, sourceuri, extid) for the entity with id <eid>"""
raise NotImplementedError
......
......@@ -154,7 +154,7 @@ class Repository(object):
self.sources_by_uri = {'system': self.system_source}
# querier helper, need to be created after sources initialization
self.querier = querier.QuerierHelper(self, self.schema)
# cache eid -> (type, source, extid)
# cache eid -> (type, physical source, extid, actual source)
self._type_source_cache = {}
# cache (extid, source uri) -> eid
self._extid_cache = {}
......@@ -534,7 +534,7 @@ class Repository(object):
# XXX we may want to check we don't give sensible information
if foreid is None:
return self.config[option]
_, sourceuri, extid = self.type_and_source_from_eid(foreid)
_, sourceuri, extid, _ = self.type_and_source_from_eid(foreid)
if sourceuri == 'system':
return self.config[option]
cnxset = self._get_cnxset()
......@@ -749,7 +749,9 @@ class Repository(object):
session.free_cnxset()
def describe(self, sessionid, eid, txid=None):
"""return a tuple (type, source, extid) for the entity with id <eid>"""
"""return a tuple `(type, physical source uri, extid, actual source
uri)` for the entity of the given `eid`
"""
session = self._get_session(sessionid, setcnxset=True, txid=txid)
try:
return self.type_and_source_from_eid(eid, session)
......@@ -954,7 +956,9 @@ class Repository(object):
# * correspondance between eid and local id (i.e. specific to a given source)
def type_and_source_from_eid(self, eid, session=None):
"""return a tuple (type, source, extid) for the entity with id <eid>"""
"""return a tuple `(type, physical source uri, extid, actual source
uri)` for the entity of the given `eid`
"""
try:
eid = typed_eid(eid)
except ValueError:
......@@ -968,15 +972,15 @@ class Repository(object):
else:
free_cnxset = False
try:
etype, uri, extid = self.system_source.eid_type_source(session,
eid)
etype, uri, extid, auri = self.system_source.eid_type_source(
session, eid)
finally:
if free_cnxset:
session.free_cnxset()
self._type_source_cache[eid] = (etype, uri, extid)
if uri != 'system':
self._extid_cache[(extid, uri)] = eid
return etype, uri, extid
self._type_source_cache[eid] = (etype, uri, extid, auri)
if uri != 'system':
self._extid_cache[(extid, uri)] = eid
return etype, uri, extid, auri
def clear_caches(self, eids):
etcache = self._type_source_cache
......@@ -984,7 +988,7 @@ class Repository(object):
rqlcache = self.querier._rql_cache
for eid in eids:
try:
etype, uri, extid = etcache.pop(typed_eid(eid)) # may be a string in some cases
etype, uri, extid, auri = etcache.pop(typed_eid(eid)) # may be a string in some cases
rqlcache.pop('%s X WHERE X eid %s' % (etype, eid), None)
extidcache.pop((extid, uri), None)
except KeyError:
......@@ -1018,7 +1022,7 @@ class Repository(object):
def eid2extid(self, source, eid, session=None):
"""get local id from an eid"""
etype, uri, extid = self.type_and_source_from_eid(eid, session)
etype, uri, extid, _ = self.type_and_source_from_eid(eid, session)
if source.uri != uri:
# eid not from the given source
raise UnknownEid(eid)
......@@ -1061,7 +1065,7 @@ class Repository(object):
eid = self.system_source.extid2eid(session, uri, extid)
if eid is not None:
self._extid_cache[cachekey] = eid
self._type_source_cache[eid] = (etype, uri, extid)
self._type_source_cache[eid] = (etype, uri, extid, source.uri)
if free_cnxset:
session.free_cnxset()
return eid
......@@ -1079,7 +1083,7 @@ class Repository(object):
try:
eid = self.system_source.create_eid(session)
self._extid_cache[cachekey] = eid
self._type_source_cache[eid] = (etype, uri, extid)
self._type_source_cache[eid] = (etype, uri, extid, source.uri)
entity = source.before_entity_insertion(
session, extid, etype, eid, sourceparams)
if source.should_call_hooks:
......@@ -1215,7 +1219,8 @@ class Repository(object):
suri = 'system'
extid = source.get_extid(entity)
self._extid_cache[(str(extid), suri)] = entity.eid
self._type_source_cache[entity.eid] = (entity.__regid__, suri, extid)
self._type_source_cache[entity.eid] = (entity.__regid__, suri, extid,
source.uri)
return extid
def glob_add_entity(self, session, edited):
......@@ -1376,7 +1381,7 @@ class Repository(object):
# in setdefault, this should not be changed without profiling.
for eid in eids:
etype, sourceuri, extid = self.type_and_source_from_eid(eid, session)
etype, sourceuri, extid, _ = self.type_and_source_from_eid(eid, session)
# XXX should cache entity's cw_metainformation
entity = session.entity_from_eid(eid, etype)
try:
......
......@@ -138,8 +138,8 @@ def deserialize_schema(schema, session):
except:
pass
tocleanup = [eid]
tocleanup += (eid for eid, (eidetype, uri, extid) in repo._type_source_cache.items()
if etype == eidetype)
tocleanup += (eid for eid, cached in repo._type_source_cache.iteritems()
if etype == cached[0])
repo.clear_caches(tocleanup)
session.commit(False)
if needcopy:
......
......@@ -870,9 +870,13 @@ class Session(RequestSessionBase):
def source_defs(self):
return self.repo.source_defs()
def describe(self, eid):
def describe(self, eid, asdict=False):
"""return a tuple (type, sourceuri, extid) for the entity with id <eid>"""
return self.repo.type_and_source_from_eid(eid, self)
metas = self.repo.type_and_source_from_eid(eid, self)
if asdict:
return dict(zip(('type', 'source', 'extid', 'asource'), metas))
# XXX :-1 for cw compat, use asdict=True for full information
return metas[:-1]
# db-api like interface ###################################################
......
......@@ -411,6 +411,10 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
def init(self, activated, source_entity):
self.init_creating(source_entity._cw.cnxset)
try:
source_entity._cw.system_sql('SELECT COUNT(asource) FROM entities')
except Exception, ex:
self.eid_type_source = self.eid_type_source_pre_131
def shutdown(self):
if self._eid_creation_cnx:
......@@ -677,7 +681,7 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
for subject, object in subj_obj_list:
self._record_tx_action(session, 'tx_relation_actions', 'A',
eid_from=subject, rtype=rtype, eid_to=object)
def _add_relations(self, session, rtype, subj_obj_list, inlined=False):
"""add a relation to the source"""
sql = []
......@@ -846,7 +850,7 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
def eid_type_source(self, session, eid):
"""return a tuple (type, source, extid) for the entity with id <eid>"""
sql = 'SELECT type, source, extid FROM entities WHERE eid=%s' % eid
sql = 'SELECT type, source, extid, asource FROM entities WHERE eid=%s' % eid
try:
res = self.doexec(session, sql).fetchone()
except:
......@@ -854,10 +858,27 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
raise UnknownEid(eid)
if res is None:
raise UnknownEid(eid)
if res[-1] is not None:
if res[-2] is not None:
if not isinstance(res, list):
res = list(res)
res[-2] = b64decode(res[-2])
return res
def eid_type_source_pre_131(self, session, eid):
"""return a tuple (type, source, extid) for the entity with id <eid>"""
sql = 'SELECT type, source, extid FROM entities WHERE eid=%s' % eid
try:
res = self.doexec(session, sql).fetchone()
except:
assert session.cnxset, 'session has no connections set'
raise UnknownEid(eid)
if res is None:
raise UnknownEid(eid)
if not isinstance(res, list):
res = list(res)
if res[-1] is not None:
res[-1] = b64decode(res[-1])
res.append(res[1])
return res
def extid2eid(self, session, source_uri, extid):
......@@ -946,7 +967,7 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
extid = b64encode(extid)
uri = 'system' if source.copy_based_source else source.uri
attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': extid,
'source': uri, 'mtime': datetime.now()}
'source': uri, 'asource': source.uri, 'mtime': datetime.now()}
self.doexec(session, self.sqlgen.insert('entities', attrs), attrs)
# insert core relations: is, is_instance_of and cw_source
try:
......@@ -1434,6 +1455,7 @@ CREATE TABLE entities (
eid INTEGER PRIMARY KEY NOT NULL,
type VARCHAR(64) NOT NULL,
source VARCHAR(64) NOT NULL,
asource VARCHAR(64) NOT NULL,
mtime %s NOT NULL,
extid VARCHAR(256)
);;
......
......@@ -64,12 +64,12 @@ class DataFeedTC(CubicWebTC):
self.assertEqual(entity.cw_source[0].name, 'myfeed')
self.assertEqual(entity.cw_metainformation(),
{'type': 'Card',
'source': {'uri': 'system', 'type': 'native'},
'source': {'uri': 'myfeed', 'type': 'datafeed'},
'extid': 'http://www.cubicweb.org/'}
)
# test repo cache keys
self.assertEqual(self.repo._type_source_cache[entity.eid],
('Card', 'system', 'http://www.cubicweb.org/'))
('Card', 'system', 'http://www.cubicweb.org/', 'myfeed'))
self.assertEqual(self.repo._extid_cache[('http://www.cubicweb.org/', 'system')],
entity.eid)
# test repull
......@@ -83,7 +83,7 @@ class DataFeedTC(CubicWebTC):
self.assertEqual(stats['created'], set())
self.assertEqual(stats['updated'], set((entity.eid,)))
self.assertEqual(self.repo._type_source_cache[entity.eid],
('Card', 'system', 'http://www.cubicweb.org/'))
('Card', 'system', 'http://www.cubicweb.org/', 'myfeed'))
self.assertEqual(self.repo._extid_cache[('http://www.cubicweb.org/', 'system')],
entity.eid)
......
This diff is collapsed.
......@@ -385,7 +385,7 @@ class RepositoryTC(CubicWebTC):
cnxid = repo.connect(self.admlogin, password=self.admpassword)
session = repo._get_session(cnxid, setcnxset=True)
self.assertEqual(repo.type_and_source_from_eid(2, session),
('CWGroup', 'system', None))
('CWGroup', 'system', None, 'system'))
self.assertEqual(repo.type_from_eid(2, session), 'CWGroup')
self.assertEqual(repo.source_from_eid(2, session).uri, 'system')
self.assertEqual(repo.eid2extid(repo.system_source, 2, session), None)
......@@ -403,7 +403,7 @@ class RepositoryTC(CubicWebTC):
repo = self.repo
cnxid = repo.connect(self.admlogin, password=self.admpassword)
self.assertEqual(repo.user_info(cnxid), (6, 'admin', set([u'managers']), {}))
self.assertEqual(repo.describe(cnxid, 2), (u'CWGroup', u'system', None))
self.assertEqual(repo.describe(cnxid, 2), (u'CWGroup', u'system', None, 'system'))
repo.close(cnxid)
self.assertRaises(BadConnectionId, repo.user_info, cnxid)
self.assertRaises(BadConnectionId, repo.describe, cnxid, 1)
......@@ -548,10 +548,11 @@ class DataHelpersTC(CubicWebTC):
self.repo.add_info(self.session, entity, self.repo.system_source)
cu = self.session.system_sql('SELECT * FROM entities WHERE eid = -1')
data = cu.fetchall()
self.assertIsInstance(data[0][3], datetime)
self.assertIsInstance(data[0][4], datetime)
data[0] = list(data[0])
data[0][3] = None
self.assertEqual(tuplify(data), [(-1, 'Personne', 'system', None, None)])
data[0][4] = None
self.assertEqual(tuplify(data), [(-1, 'Personne', 'system', 'system',
None, None)])
self.repo.delete_info(self.session, entity, 'system', None)
#self.repo.commit()
cu = self.session.system_sql('SELECT * FROM entities WHERE eid = -1')
......@@ -813,6 +814,7 @@ class PerformanceTest(CubicWebTC):
req.cnx.commit()
t1 = time.time()
self.info('add relations: %.2gs', t1-t0)
def test_session_add_relation_inlined(self):
""" to be compared with test_session_add_relations"""
req = self.request()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment