Commit 96775afb authored by Philippe Pepiot's avatar Philippe Pepiot
Browse files

[cwvreg] load registry using modules names instead of directories

Introspect cubicweb, cubes and apphome using pkgutil to generate the full list
of modules names for loading registries.
Avoiding using bogus logilab.common.modutils.modpath_from_file().
parent bf6106b91633
......@@ -823,57 +823,12 @@ this option is set to yes",
modnames.append(('data', modname))
return modnames
def appobjects_path(self):
"""return a list of files or directories where the registry will look
for application objects. By default return nothing in NoApp config.
def appobjects_modnames(self):
"""return a list of modules where the registry will look for
application objects. By default return nothing in NoApp config.
return []
def build_appobjects_path(self, templpath, evobjpath=None, tvobjpath=None):
"""given a list of directories, return a list of sub files and
directories that should be loaded by the instance objects registry.
:param evobjpath:
optional list of sub-directories (or files without the .py ext) of
the cubicweb library that should be tested and added to the output list
if they exists. If not give, default to `cubicweb_appobject_path` class
:param tvobjpath:
optional list of sub-directories (or files without the .py ext) of
directories given in `templpath` that should be tested and added to
the output list if they exists. If not give, default to
`cube_appobject_path` class attribute.
vregpath = self.build_appobjects_cubicweb_path(evobjpath)
vregpath += self.build_appobjects_cube_path(templpath, tvobjpath)
return vregpath
def build_appobjects_cubicweb_path(self, evobjpath=None):
vregpath = []
if evobjpath is None:
evobjpath = self.cubicweb_appobject_path
# NOTE: for the order, see
# it is clearly a workaround
for subdir in sorted(evobjpath, key=lambda x:x != 'entities'):
path = join(CW_SOFTWARE_ROOT, subdir)
if exists(path):
return vregpath
def build_appobjects_cube_path(self, templpath, tvobjpath=None):
vregpath = []
if tvobjpath is None:
tvobjpath = self.cube_appobject_path
for directory in templpath:
# NOTE: for the order, see
for subdir in sorted(tvobjpath, key=lambda x:x != 'entities'):
path = join(directory, subdir)
if exists(path):
elif exists(path + '.py'):
vregpath.append(path + '.py')
return vregpath
apphome = None
def load_site_cubicweb(self, cubes=()):
......@@ -1361,14 +1316,42 @@ the repository',
self.exception('localisation support error for language %s',
def appobjects_path(self):
"""return a list of files or directories where the registry will look
for application objects
templpath = list(reversed(self.cubes_path()))
if self.apphome: # may be unset in tests
return self.build_appobjects_path(templpath)
def _sorted_appobjects(appobjects):
appobjects = sorted(appobjects)
index = appobjects.index('entities')
except ValueError:
# put entities first
appobjects.insert(0, appobjects.pop(index))
return appobjects
def _appobjects_cube_modnames(self, cube):
modnames = []
cube_submodnames = self._sorted_appobjects(self.cube_appobject_path)
for name in cube_submodnames:
for modname, filepath in _expand_modname('.'.join(['cubes', cube, name])):
return modnames
def appobjects_modnames(self):
modnames = []
for name in self._sorted_appobjects(self.cubicweb_appobject_path):
for modname, filepath in _expand_modname('cubicweb.' + name):
for cube in reversed(self.cubes()):
if self.apphome:
cube_submodnames = self._sorted_appobjects(self.cube_appobject_path)
apphome = realpath(self.apphome)
for name in cube_submodnames:
for modname, filepath in _expand_modname(name):
# ensure file is in apphome
if realpath(filepath).startswith(apphome):
return modnames
def set_sources_mode(self, sources):
if not 'all' in sources:
......@@ -29,7 +29,7 @@ from six import text_type, binary_type
from logilab.common.decorators import cached, clear_cache
from logilab.common.deprecation import class_deprecated
from logilab.common.modutils import cleanup_sys_modules
from logilab.common.modutils import clean_sys_modules
from logilab.common.registry import RegistryStore, Registry, ObjectNotFound, RegistryNotFound
from rql import RQLHelper
......@@ -417,7 +417,7 @@ class CWRegistryStore(RegistryStore):
"""set instance'schema and load application objects"""
# now we can load application's web objects
self.reload(self.config.appobjects_path(), force_reload=False)
self.reload(self.config.appobjects_modnames(), force_reload=False)
# map lowered entity type names to their actual name
self.case_insensitive_etypes = {}
for eschema in self.schema.entities():
......@@ -426,13 +426,28 @@ class CWRegistryStore(RegistryStore):
clear_cache(eschema, 'ordered_relations')
clear_cache(eschema, 'meta_attributes')
def is_reload_needed(self, modnames):
"""overriden to handle modules names instead of directories"""
lastmodifs = self._lastmodifs
for modname in modnames:
if modname not in sys.modules:
# new module to load
return True
filepath = sys.modules[modname].__file__
if filepath.endswith('.py'):
mdate = self._mdate(filepath)
if filepath not in lastmodifs or lastmodifs[filepath] < mdate:'File %s changed since last visit', filepath)
return True
return False
def reload_if_needed(self):
path = self.config.appobjects_path()
if self.is_reload_needed(path):
modnames = self.config.appobjects_modnames()
if self.is_reload_needed(modnames):
def _cleanup_sys_modules(self, path):
"""Remove submodules of `directories` from `sys.modules` and cleanup
def _cleanup_sys_modules(self, modnames):
"""Remove modules and submodules of `modnames` from `sys.modules` and cleanup
CW_EVENT_MANAGER accordingly.
We take care to properly remove obsolete registry callbacks.
......@@ -446,18 +461,18 @@ class CWRegistryStore(RegistryStore):
# for non-function callable, we do nothing interesting
module = getattr(func, '__module__', None)
caches[id(callback)] = module
deleted_modules = set(cleanup_sys_modules(path))
deleted_modules = set(clean_sys_modules(modnames))
for callbacklist in callbackdata:
for callback in callbacklist[:]:
module = caches[id(callback)]
if module and module in deleted_modules:
def reload(self, path, force_reload=True):
def reload(self, modnames, force_reload=True):
"""modification detected, reset and reload the vreg"""
if force_reload:
cubes = self.config.cubes()
# if the fs code use some cubes not yet registered into the instance
# we should cleanup sys.modules for those as well to avoid potential
......@@ -465,9 +480,9 @@ class CWRegistryStore(RegistryStore):
cfg = self.config
for cube in cfg.expand_cubes(cubes, with_recommends=True):
if not cube in cubes:
cpath = cfg.build_appobjects_cube_path([cfg.cube_dir(cube)])
cube_modnames = cfg.appobjects_cube_modnames(cube)
def load_file(self, filepath, modname):
......@@ -37,6 +37,7 @@ from pytz import UTC
from six.moves import input
from logilab.common import STD_BLACKLIST
from logilab.common.modutils import clean_sys_modules
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find
......@@ -100,24 +101,6 @@ class DevConfiguration(ServerConfiguration, WebConfiguration):
return None
def cleanup_sys_modules(config):
# cleanup sys.modules, required when we're updating multiple cubes
appobjects_path = config.appobjects_path()
for name, mod in list(sys.modules.items()):
if mod is None:
# duh ? logilab.common.os for instance
del sys.modules[name]
if not hasattr(mod, '__file__'):
if mod.__file__ is None:
# odd/rare but real
for path in appobjects_path:
if mod.__file__.startswith(path):
del sys.modules[name]
def generate_schema_pot(w, cubedir=None):
"""generate a pot file with schema specific i18n messages
......@@ -136,7 +119,7 @@ def generate_schema_pot(w, cubedir=None):
config = DevConfiguration()
cube = libconfig = None
schema = config.load_schema(remove_unused_rtypes=False)
vreg = CWRegistryStore(config)
# set_schema triggers objects registrations
......@@ -161,7 +144,7 @@ def _generate_schema_pot(w, vreg, schema, libconfig=None):
# (cubicweb incl.)
from cubicweb.cwvreg import CWRegistryStore
libschema = libconfig.load_schema(remove_unused_rtypes=False)
libvreg = CWRegistryStore(libconfig)
libvreg.set_schema(libschema) # trigger objects registration
libafss = libvreg['uicfg']['autoform_section']
......@@ -178,17 +178,6 @@ class CubicWebConfigurationTC(BaseTestCase):
self.assertEqual(self.config.expand_cubes(('email', 'comment')),
['email', 'comment', 'file'])
def test_appobjects_path(self):
path = [unabsolutize(p) for p in self.config.appobjects_path()]
self.assertEqual(path[0], 'entities')
self.assertCountEqual(path[1:4], ['web/views', 'sobjects', 'hooks'])
self.assertEqual(path[4], 'file/entities')
['file/', 'file/hooks'])
self.assertEqual(path[7], 'email/')
['email/views', 'email/'])
self.assertEqual(path[10:], ['test/data/', 'test/data/'])
def test_init_cubes_ignore_pyramid_cube(self):
warning_msg = 'cubicweb-pyramid got integrated into CubicWeb'
......@@ -463,7 +452,48 @@ class ModnamesTC(unittest.TestCase):
join(libdir, ''))
self.assertEqual(config.schema_modnames(), expected)
def test_appobjects_modnames(self, libdir):
for filepath in (
join(libdir, ''),
join(libdir, 'cubicweb_foo', ''),
join(libdir, 'cubicweb_foo', 'entities', ''),
join(libdir, 'cubicweb_foo', 'entities', ''),
join(libdir, 'cubicweb_foo', ''),
join(libdir, 'cubes', ''),
join(libdir, 'cubes', 'bar', ''),
join(libdir, 'cubes', 'bar', ''),
join(libdir, '_instance_dir', 'data1', ''),
join(libdir, '_instance_dir', 'data2', ''),
instance_dir, cubes_dir = (
join(libdir, '_instance_dir'), join(libdir, 'cubes'))
expected = [
# data1 has entities
with temp_config('data1', instance_dir, cubes_dir,
('foo', 'bar')) as config:
config.cube_appobject_path = set(['entities', 'hooks'])
expected + ['entities'])
# data2 has hooks
with temp_config('data2', instance_dir, cubes_dir,
('foo', 'bar')) as config:
config.cube_appobject_path = set(['entities', 'hooks'])
expected + ['hooks'])
if __name__ == '__main__':
......@@ -82,7 +82,7 @@ class WebConfiguration(CubicWebConfiguration):
"""the WebConfiguration is a singleton object handling instance's
configuration and preferences
cubicweb_appobject_path = CubicWebConfiguration.cubicweb_appobject_path | set([join('web', 'views')])
cubicweb_appobject_path = CubicWebConfiguration.cubicweb_appobject_path | set(['web.views'])
cube_appobject_path = CubicWebConfiguration.cube_appobject_path | set(['views'])
options = merge_options(CubicWebConfiguration.options + (
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment