Commit ce388ff9 authored by Denis Laxalde's avatar Denis Laxalde
Browse files

[server] Add source_by_eid and source_by_uri methods to repository

Most of the times we only need to retrieve one source (either by uri or eid)
and querying sources_by_eid and sources_by_uri properties on repository just
for one item is costly. So these methods query what's needed. We issue a
ValueError (instead of KeyError for sources_by_{eid,uri} dict) in case the key
is not found.

--HG--
branch : 3.25
parent 752b94ed9748
......@@ -45,7 +45,7 @@ class MassiveObjectStoreWithCustomMDGenStoreTC(
def store_impl(self, cnx):
source = cnx.create_entity('CWSource', type=u'datafeed', name=u'test', url=u'test')
cnx.commit()
metagen = stores.MetadataGenerator(cnx, source=cnx.repo.sources_by_eid[source.eid])
metagen = stores.MetadataGenerator(cnx, source=cnx.repo.source_by_eid(source.eid))
return MassiveObjectStore(cnx, metagen=metagen)
......
......@@ -80,7 +80,7 @@ class NoHookRQLObjectStoreWithCustomMDGenStoreTC(RQLObjectStoreTC):
def store_impl(self, cnx):
source = cnx.create_entity('CWSource', type=u'datafeed', name=u'test', url=u'test')
cnx.commit()
metagen = stores.MetadataGenerator(cnx, source=cnx.repo.sources_by_eid[source.eid])
metagen = stores.MetadataGenerator(cnx, source=cnx.repo.source_by_eid(source.eid))
return stores.NoHookRQLObjectStore(cnx, metagen)
......
......@@ -63,7 +63,7 @@ class CWSource(_CWSourceCfgMixIn, AnyEntity):
"""repository only property, not available from the web side (eg
self._cw is expected to be a server session)
"""
return self._cw.repo.sources_by_eid[self.eid]
return self._cw.repo.source_by_eid(self.eid)
class CWSourceHostConfig(_CWSourceCfgMixIn, AnyEntity):
......
......@@ -8,7 +8,7 @@ except ValueError:
print()
print('you should not have updated your sources file yet')
olddn = repo.sources_by_uri[uri].config['user-base-dn']
olddn = repo.source_by_uri(uri).config['user-base-dn']
assert olddn != newdn
......
......@@ -302,6 +302,20 @@ class Repository(object):
# 5. call instance level initialisation hooks
self.hm.call_hooks('server_startup', repo=self)
def source_by_uri(self, uri):
with self.internal_cnx() as cnx:
rset = cnx.find('CWSource', name=uri)
if not rset:
raise ValueError('no source with uri %s found' % uri)
return self._source_from_cwsource(rset.one())
def source_by_eid(self, eid):
with self.internal_cnx() as cnx:
rset = cnx.find('CWSource', eid=eid)
if not rset:
raise ValueError('no source with eid %d found' % eid)
return self._source_from_cwsource(rset.one())
@property
def sources_by_uri(self):
mapping = {'system': self.system_source}
......@@ -323,20 +337,24 @@ class Repository(object):
for sourceent in cnx.execute(
'Any S, SN, SA, SC WHERE S is_instance_of CWSource, '
'S name SN, S type SA, S config SC, S name != "system"').entities():
source = self.get_source(sourceent.type, sourceent.name,
sourceent.host_config, sourceent.eid)
if self.config.source_enabled(source):
# call source's init method to complete their initialisation if
# needed (for instance looking for persistent configuration using an
# internal session, which is not possible until connections sets have been
# initialized)
source.init(True, sourceent)
else:
source.init(False, sourceent)
source.set_schema(self.schema)
source = self._source_from_cwsource(sourceent)
yield sourceent, source
self._clear_source_defs_caches()
def _source_from_cwsource(self, sourceent):
source = self.get_source(sourceent.type, sourceent.name,
sourceent.host_config, sourceent.eid)
if self.config.source_enabled(source):
# call source's init method to complete their initialisation if
# needed (for instance looking for persistent configuration using an
# internal session, which is not possible until connections sets have been
# initialized)
source.init(True, sourceent)
else:
source.init(False, sourceent)
source.set_schema(self.schema)
return source
# internals ###############################################################
def _init_system_source(self):
......
......@@ -1056,8 +1056,8 @@ class SynchronizeSourceCommand(Command):
if len(args) >= 2:
for name in args[1:]:
try:
source = repo.sources_by_uri[name]
except KeyError:
source = repo.source_by_uri(name)
except ValueError:
cnx.error('no source named %r' % name)
errors = True
else:
......
......@@ -191,7 +191,7 @@ class DataFeedSource(AbstractSource):
def _synchronize_source(repo, source_eid, import_log_eid):
with repo.internal_cnx() as cnx:
source = repo.sources_by_eid[source_eid]
source = repo.source_by_eid(source_eid)
source._pull_data(cnx, force, raise_on_error, import_log_eid=import_log_eid)
sync = partial(_synchronize_source, cnx.repo, self.eid, import_log.eid)
......
......@@ -50,18 +50,19 @@ class DataFeedTC(CubicWebTC):
store.commit()
with self.temporary_appobjects(AParser):
if u'ô myfeed' in self.repo.sources_by_uri:
yield self.repo.sources_by_uri[u'ô myfeed']._get_parser(session)
else:
try:
source = self.repo.source_by_uri(u'ô myfeed')
except ValueError:
yield
else:
yield source._get_parser(session)
# vreg.unregister just pops appobjects from their regid entry,
# completely remove the entry to ensure we have no side effect with
# this empty entry.
del self.vreg['parsers'][AParser.__regid__]
def test(self):
self.assertIn(u'ô myfeed', self.repo.sources_by_uri)
dfsource = self.repo.sources_by_uri[u'ô myfeed']
dfsource = self.repo.source_by_uri(u'ô myfeed')
self.assertNotIn('use_cwuri_as_url', dfsource.__dict__)
self.assertEqual({'type': u'datafeed', 'uri': u'ô myfeed', 'use-cwuri-as-url': True},
dfsource.public_config)
......@@ -113,16 +114,16 @@ class DataFeedTC(CubicWebTC):
self.assertEqual('a string', value.geturl())
def test_update_url(self):
dfsource = self.repo.sources_by_uri[u'ô myfeed']
dfsource = self.repo.source_by_uri(u'ô myfeed')
with self.admin_access.repo_cnx() as cnx:
cnx.entity_from_eid(dfsource.eid).cw_set(url=u"http://pouet.com\nhttp://pouet.org")
cnx.commit()
self.assertEqual(dfsource.urls, [u'ignored'])
dfsource = self.repo.sources_by_uri[u'ô myfeed']
dfsource = self.repo.source_by_uri(u'ô myfeed')
self.assertEqual(dfsource.urls, [u"http://pouet.com", u"http://pouet.org"])
def test_parser_not_found(self):
dfsource = self.repo.sources_by_uri[u'ô myfeed']
dfsource = self.repo.source_by_uri(u'ô myfeed')
with self.assertLogs('cubicweb.sources.o myfeed', level='ERROR') as cm:
with self.repo.internal_cnx() as cnx:
stats = dfsource.pull_data(cnx, force=True)
......@@ -141,7 +142,7 @@ class DataFeedConfigTC(CubicWebTC):
parser=u'testparser', url=u'ignored',
config=u'use-cwuri-as-url=no')
cnx.commit()
dfsource = self.repo.sources_by_uri['myfeed']
dfsource = self.repo.source_by_uri('myfeed')
self.assertEqual(dfsource.use_cwuri_as_url, False)
self.assertEqual({'type': u'datafeed', 'uri': u'myfeed', 'use-cwuri-as-url': False},
dfsource.public_config)
......
......@@ -159,7 +159,7 @@ class LDAPFeedTestBase(CubicWebTC):
@staticmethod
def pull(cnx):
lfsource = cnx.repo.sources_by_uri['ldap']
lfsource = cnx.repo.source_by_uri('ldap')
stats = lfsource.pull_data(cnx, force=True, raise_on_error=True)
cnx.commit()
return stats
......@@ -208,7 +208,7 @@ class LDAPFeedTestBase(CubicWebTC):
self._ldapmodify(modcmd)
def _ldapmodify(self, modcmd):
uri = self.repo.sources_by_uri['ldap'].urls[0]
uri = self.repo.source_by_uri('ldap').urls[0]
updatecmd = ['ldapmodify', '-H', uri, '-v', '-x', '-D',
'cn=admin,dc=cubicweb,dc=test', '-w', 'cw']
PIPE = subprocess.PIPE
......@@ -247,7 +247,7 @@ class LDAPFeedUserTC(LDAPFeedTestBase):
self.assertTrue(entity.modification_date)
def test_authenticate(self):
source = self.repo.sources_by_uri['ldap']
source = self.repo.source_by_uri('ldap')
with self.admin_access.repo_cnx() as cnx:
# ensure we won't be logged against
self.assertRaises(AuthenticationError,
......@@ -282,7 +282,7 @@ class LDAPFeedUserTC(LDAPFeedTestBase):
def test_copy_to_system_source(self):
"make sure we can 'convert' an LDAP user into a system one"
with self.admin_access.repo_cnx() as cnx:
source = self.repo.sources_by_uri['ldap']
source = self.repo.source_by_uri('ldap')
eid = cnx.execute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0]
cnx.execute('SET X cw_source S WHERE X eid %(x)s, S name "system"', {'x': eid})
cnx.commit()
......@@ -315,7 +315,7 @@ class LDAPGeneratePwdTC(LDAPFeedTestBase):
def setup_database(self):
with self.admin_access.repo_cnx() as cnx:
lfsource = cnx.repo.sources_by_uri['ldap']
lfsource = cnx.repo.source_by_uri('ldap')
del lfsource.user_attrs['userPassword']
super(LDAPGeneratePwdTC, self).setup_database()
......@@ -342,7 +342,7 @@ class LDAPFeedUserDeletionTC(LDAPFeedTestBase):
cnx.commit()
with self.repo.internal_cnx() as cnx:
self.pull(cnx)
repo_source = self.repo.sources_by_uri['ldap']
repo_source = self.repo.source_by_uri('ldap')
self.assertRaises(AuthenticationError,
repo_source.authenticate, cnx, 'syt', 'syt')
with self.admin_access.repo_cnx() as cnx:
......@@ -374,7 +374,7 @@ class LDAPFeedUserDeletionTC(LDAPFeedTestBase):
self.delete_ldap_entry('uid=syt,ou=People,dc=cubicweb,dc=test')
with self.repo.internal_cnx() as cnx:
self.pull(cnx)
source = self.repo.sources_by_uri['ldap']
source = self.repo.source_by_uri('ldap')
self.assertRaises(AuthenticationError,
source.authenticate, cnx, 'syt', 'syt')
with self.admin_access.repo_cnx() as cnx:
......@@ -413,7 +413,7 @@ class LDAPFeedUserDeletionTC(LDAPFeedTestBase):
# test reactivating BY HAND the user isn't enough to
# authenticate, as the native source refuse to authenticate
# user from other sources
repo_source = self.repo.sources_by_uri['ldap']
repo_source = self.repo.source_by_uri('ldap')
self.delete_ldap_entry('uid=syt,ou=People,dc=cubicweb,dc=test')
with self.repo.internal_cnx() as cnx:
self.pull(cnx)
......
......@@ -52,7 +52,7 @@ class PostgresFTITC(CubicWebTC):
def test_eid_range(self):
# concurrent allocation of eid ranges
source = self.repo.sources_by_uri['system']
source = self.repo.system_source
range1 = []
range2 = []
def allocate_eid_ranges(session, target):
......
......@@ -378,7 +378,7 @@ class UndoableTransactionTC(CubicWebTC):
txuuid = cnx.commit()
p = cnx.create_entity('Personne', nom=u'louis', fiche=c)
cnx.commit()
integrityerror = self.repo.sources_by_uri['system'].dbhelper.dbapi_module.IntegrityError
integrityerror = self.repo.system_source.dbhelper.dbapi_module.IntegrityError
with self.assertRaises(integrityerror):
cnx.undo_transaction(txuuid)
......
......@@ -142,6 +142,6 @@ class SourceSynchronizationService(Service):
__select__ = Service.__select__ & match_user_groups('managers')
def call(self, source_eid):
source = self._cw.repo.sources_by_eid[source_eid]
source = self._cw.repo.source_by_eid(source_eid)
result = source.pull_data(self._cw, force=True, async=True)
return result['import_log_eid']
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment