merge

appobject.py

@@ -377,8 +377,8 @@ class AppObject(object):
         # try to get page boundaries from the navigation component
         # XXX we should probably not have a ref to this component here (eg in
         #     cubicweb.common)
-        nav = self.vreg['components'].select_object('navigation', self.req,
-                                                    rset=self.rset)
+        nav = self.vreg['components'].select_or_none('navigation', self.req,
+                                                     rset=self.rset)
         if nav:
             start, stop = nav.page_boundaries()
             rql = self._limit_offset_rql(stop - start, start)

cwvreg.py

@@ -69,7 +69,7 @@ class CWRegistry(Registry):
         return obj.render(**kwargs)
 
     def select_vobject(self, oid, *args, **kwargs):
-        selected = self.select_object(oid, *args, **kwargs)
+        selected = self.select_or_none(oid, *args, **kwargs)
         if selected and selected.propval('visible'):
             return selected
         return None
@@ -172,7 +172,7 @@ class ViewsRegistry(CWRegistry):
             if vid[0] == '_':
                 continue
             try:
-                view = self.select_best(views, req, rset=rset, **kwargs)
+                view = self._select_best(views, req, rset=rset, **kwargs)
                 if view.linkable():
                     yield view
             except NoSelectableObject:
@@ -403,17 +403,17 @@ class CubicWebVRegistry(VRegistry):
     def possible_actions(self, req, rset=None, **kwargs):
         return self["actions"].possible_actions(req, rest=rset, **kwargs)
 
-    @deprecated("use vreg['boxes'].select_object(...)")
+    @deprecated("use vreg['boxes'].select_or_none(...)")
     def select_box(self, oid, *args, **kwargs):
-        return self['boxes'].select_object(oid, *args, **kwargs)
+        return self['boxes'].select_or_none(oid, *args, **kwargs)
 
-    @deprecated("use vreg['components'].select_object(...)")
+    @deprecated("use vreg['components'].select_or_none(...)")
     def select_component(self, cid, *args, **kwargs):
-        return self['components'].select_object(cid, *args, **kwargs)
+        return self['components'].select_or_none(cid, *args, **kwargs)
 
-    @deprecated("use vreg['actions'].select_object(...)")
+    @deprecated("use vreg['actions'].select_or_none(...)")
     def select_action(self, oid, *args, **kwargs):
-        return self['actions'].select_object(oid, *args, **kwargs)
+        return self['actions'].select_or_none(oid, *args, **kwargs)
 
     @deprecated("use vreg['views'].select(...)")
     def select_view(self, __vid, req, rset=None, **kwargs):

dbapi.py

@@ -42,9 +42,9 @@ def multiple_connections_fix():
     registries.
     """
     defaultcls = cwvreg.VRegistry.REGISTRY_FACTORY[None]
-    orig_select_best = defaultcls.orig_select_best = defaultcls.select_best
+    orig_select_best = defaultcls.orig_select_best = defaultcls._select_best
     @monkeypatch(defaultcls)
-    def select_best(self, appobjects, *args, **kwargs):
+    def _select_best(self, appobjects, *args, **kwargs):
         """return an instance of the most specific object according
         to parameters
 

devtools/testlib.py

@@ -284,7 +284,7 @@ class WebTest(EnvBasedTC):
                      and not issubclass(view, NotificationView)]
             if views:
                 try:
-                    view = viewsvreg.select_best(views, req, rset=rset)
+                    view = viewsvreg._select_best(views, req, rset=rset)
                     if view.linkable():
                         yield view
                     else:
@@ -392,7 +392,7 @@ def vreg_instrumentize(testclass):
         try:
             orig_select_best = reg.__class__.__orig_select_best
         except:
-            orig_select_best = reg.__class__.select_best
+            orig_select_best = reg.__class__._select_best
         def instr_select_best(self, *args, **kwargs):
             selected = orig_select_best(self, *args, **kwargs)
             try:
@@ -402,7 +402,7 @@ def vreg_instrumentize(testclass):
             except AttributeError:
                 pass # occurs on reg used to restore database
             return selected
-        reg.__class__.select_best = instr_select_best
+        reg.__class__._select_best = instr_select_best
         reg.__class__.__orig_select_best = orig_select_best
 
 def print_untested_objects(testclass, skipregs=('hooks', 'etypes')):

etwist/server.py

@@ -116,7 +116,7 @@ class CubicWebRootResource(resource.PostableResource):
         start_task(interval, self.appli.session_handler.clean_sessions)
 
     def set_url_rewriter(self):
-        self.url_rewriter = self.appli.vreg['components'].select_object('urlrewriter')
+        self.url_rewriter = self.appli.vreg['components'].select_or_none('urlrewriter')
 
     def shutdown_event(self):
         """callback fired when the server is shutting down to properly

server/migractions.py

@@ -19,7 +19,10 @@ __docformat__ = "restructuredtext en"
 
 import sys
 import os
-from os.path import join, exists
+import tarfile
+import tempfile
+import shutil
+import os.path as osp
 from datetime import datetime
 
 from logilab.common.deprecation import deprecated
@@ -110,25 +113,77 @@ class ServerMigrationHelper(MigrationHelper):
     def backup_database(self, backupfile=None, askconfirm=True):
         config = self.config
         repo = self.repo_connect()
+        # paths
         timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        instbkdir = osp.join(config.appdatahome, 'backup')
+        if not osp.exists(instbkdir):
+            os.makedirs(instbkdir)
+        backupfile = backupfile or osp.join(instbkdir, '%s-%s.tar.gz'
+                                            % (config.appid, timestamp))
+        # check backup has to be done
+        if osp.exists(backupfile) and not \
+                self.confirm('Backup file %s exists, overwrite it?' % backupfile):
+            print '-> no backup done.'
+            return
+        elif askconfirm and not self.confirm('Backup %s database?' % config.appid):
+            print '-> no backup done.'
+            return
+        open(backupfile,'w').close() # kinda lock
+        os.chmod(backupfile, 0600)
+        # backup
+        tmpdir = tempfile.mkdtemp(dir=instbkdir)
         for source in repo.sources:
-            source.backup(self.confirm, backupfile, timestamp,
-                          askconfirm=askconfirm)
+            try:
+                source.backup(osp.join(tmpdir,source.uri))
+            except Exception, exc:
+                print '-> error trying to backup [%s]' % exc
+                if not self.confirm('Continue anyway?', default='n'):
+                    raise SystemExit(1)
+        bkup = tarfile.open(backupfile, 'w|gz')
+        for filename in os.listdir(tmpdir):
+            bkup.add(osp.join(tmpdir,filename), filename)
+        bkup.close()
+        shutil.rmtree(tmpdir)
+        # call hooks
         repo.hm.call_hooks('server_backup', repo=repo, timestamp=timestamp)
+        # done
+        print '-> backup file',  backupfile
 
     def restore_database(self, backupfile, drop=True, systemonly=True,
                          askconfirm=True):
         config = self.config
         repo = self.repo_connect()
+        # check
+        if not osp.exists(backupfile):
+            raise Exception("Backup file %s doesn't exist" % backupfile)
+            return
+        if askconfirm and not self.confirm('Restore %s database from %s ?'
+                                           % (config.appid, backupfile)):
+            return
+        # unpack backup
+        bkup = tarfile.open(backupfile, 'r|gz')
+        for name in bkup.getnames():
+            if name[0] in '/.':
+                raise Exception('Security check failed, path starts with "/" or "."')
+        bkup.close() # XXX seek error if not close+open !?!
+        bkup = tarfile.open(backupfile, 'r|gz')
+        tmpdir = tempfile.mkdtemp()
+        bkup.extractall(path=tmpdir)
         if systemonly:
-            repo.system_source.restore(self.confirm, backupfile=backupfile,
-                                       drop=drop, askconfirm=askconfirm)
+            repo.system_source.restore(osp.join(tmpdir,'system'), drop=drop)
         else:
-            # in that case, backup file is expected to be a time stamp
             for source in repo.sources:
-                source.backup(self.confirm, timestamp=backupfile, drop=drop,
-                              askconfirm=askconfirm)
-            repo.hm.call_hooks('server_restore', repo=repo, timestamp=backupfile)
+                try:
+                    source.restore(osp.join(tmpdir, source.uri), drop=drop)
+                except Exception, exc:
+                    print '-> error trying to restore [%s]' % exc
+                    if not self.confirm('Continue anyway?', default='n'):
+                        raise SystemExit(1)
+        bkup.close()
+        shutil.rmtree(tmpdir)
+        # call hooks
+        repo.hm.call_hooks('server_restore', repo=repo, timestamp=backupfile)
+        print '-> database restored.'
 
     @property
     def cnx(self):
@@ -213,10 +268,10 @@ class ServerMigrationHelper(MigrationHelper):
     def exec_event_script(self, event, cubepath=None, funcname=None,
                           *args, **kwargs):
         if cubepath:
-            apc = join(cubepath, 'migration', '%s.py' % event)
+            apc = osp.join(cubepath, 'migration', '%s.py' % event)
         else:
-            apc = join(self.config.migration_scripts_dir(), '%s.py' % event)
-        if exists(apc):
+            apc = osp.join(self.config.migration_scripts_dir(), '%s.py' % event)
+        if osp.exists(apc):
             if self.config.free_wheel:
                 from cubicweb.server.hooks import setowner_after_add_entity
                 self.repo.hm.unregister_hook(setowner_after_add_entity,

server/sources/__init__.py

@@ -95,30 +95,11 @@ class AbstractSource(object):
         """method called by the repository once ready to handle request"""
         pass
 
-    def backup_file(self, backupfile=None, timestamp=None):
-        """return a unique file name for a source's dump
-
-        either backupfile or timestamp (used to generated a backup file name if
-        needed) should be specified.
-        """
-        if backupfile is None:
-            config = self.repo.config
-            return join(config.appdatahome, 'backup',
-                        '%s-%s-%s.dump' % (config.appid, timestamp, self.uri))
-        # backup file is the system database backup file, add uri to it if not
-        # already there
-        base, ext = splitext(backupfile)
-        if not base.endswith('-%s' % self.uri):
-            return '%s-%s%s' % (base, self.uri, ext)
-        return backupfile
-
-    def backup(self, confirm, backupfile=None, timestamp=None,
-               askconfirm=False):
+    def backup(self, backupfile):
         """method called to create a backup of source's data"""
         pass
 
-    def restore(self, confirm, backupfile=None, timestamp=None, drop=True,
-               askconfirm=False):
+    def restore(self, backupfile):
         """method called to restore a backup of source's data"""
         pass
 

server/sources/extlite.py

@@ -11,8 +11,7 @@ __docformat__ = "restructuredtext en"
 from os.path import join, exists
 
 from cubicweb import server
-from cubicweb.server.sqlutils import (SQL_PREFIX, SQLAdapterMixIn, sqlexec,
-                                      sql_source_backup, sql_source_restore)
+from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn, sqlexec
 from cubicweb.server.sources import native, rql2sql
 from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results
 
@@ -94,18 +93,21 @@ repository.',
         AbstractSource.__init__(self, repo, appschema, source_config,
                                 *args, **kwargs)
 
-    def backup(self, confirm, backupfile=None, timestamp=None, askconfirm=False):
-        """method called to create a backup of source's data"""
-        backupfile = self.backup_file(backupfile, timestamp)
-        sql_source_backup(self, self.sqladapter, confirm, backupfile,
-                          askconfirm)
+    def backup(self, backupfile):
+        """method called to create a backup of the source's data"""
+        self.close_pool_connections()
+        try:
+            self.sqladapter.backup_to_file(backupfile)
+        finally:
+            self.open_pool_connections()
 
-    def restore(self, confirm, backupfile=None, timestamp=None, drop=True,
-               askconfirm=False):
+    def restore(self, backupfile, drop):
         """method called to restore a backup of source's data"""
-        backupfile = self.backup_file(backupfile, timestamp)
-        sql_source_restore(self, self.sqladapter, confirm, backupfile, drop,
-                           askconfirm)
+        self.close_pool_connections()
+        try:
+            self.sqladapter.restore_from_file(backupfile, drop)
+        finally:
+            self.open_pool_connections()
 
     @property
     def _sqlcnx(self):

server/sources/native.py

@@ -25,8 +25,7 @@ from indexer import get_indexer
 
 from cubicweb import UnknownEid, AuthenticationError, Binary, server
 from cubicweb.server.utils import crypt_password
-from cubicweb.server.sqlutils import (SQL_PREFIX, SQLAdapterMixIn,
-                                      sql_source_backup, sql_source_restore)
+from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn
 from cubicweb.server.rqlannotation import set_qdata
 from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results
 from cubicweb.server.sources.rql2sql import SQLGenerator
@@ -207,17 +206,21 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
         pool.pool_reset()
         self.repo._free_pool(pool)
 
-    def backup(self, confirm, backupfile=None, timestamp=None,
-               askconfirm=False):
-        """method called to create a backup of source's data"""
-        backupfile = self.backup_file(backupfile, timestamp)
-        sql_source_backup(self, self, confirm, backupfile, askconfirm)
+    def backup(self, backupfile):
+        """method called to create a backup of the source's data"""
+        self.close_pool_connections()
+        try:
+            self.backup_to_file(backupfile)
+        finally:
+            self.open_pool_connections()
 
-    def restore(self, confirm, backupfile=None, timestamp=None, drop=True,
-               askconfirm=False):
+    def restore(self, backupfile, drop):
         """method called to restore a backup of source's data"""
-        backupfile = self.backup_file(backupfile, timestamp)
-        sql_source_restore(self, self, confirm, backupfile, drop, askconfirm)
+        self.close_pool_connections()
+        try:
+            self.restore_from_file(backupfile, drop)
+        finally:
+            self.open_pool_connections()
 
     def init(self):
         self.init_creating()

server/sqlutils.py

@@ -120,39 +120,6 @@ def sqldropschema(schema, driver, text_index=True,
                      skip_relations=skip_relations))
     return '\n'.join(output)
 
-
-def sql_source_backup(source, sqladapter, confirm, backupfile,
-                      askconfirm=False):
-    if exists(backupfile):
-        if not confirm('Backup file %s exists, overwrite it?' % backupfile):
-            return
-    elif askconfirm and not confirm('Backup %s database?'
-                                    % source.repo.config.appid):
-        print '-> no backup done.'
-        return
-    # should close opened connection before backuping
-    source.close_pool_connections()
-    try:
-        sqladapter.backup_to_file(backupfile, confirm)
-    finally:
-        source.open_pool_connections()
-
-def sql_source_restore(source, sqladapter, confirm, backupfile, drop=True,
-                       askconfirm=False):
-    if not exists(backupfile):
-        raise Exception("backup file %s doesn't exist" % backupfile)
-    app = source.repo.config.appid
-    if askconfirm and not confirm('Restore %s %s database from %s ?'
-                                  % (app, source.uri, backupfile)):
-        return
-    # should close opened connection before restoring
-    source.close_pool_connections()
-    try:
-        sqladapter.restore_from_file(backupfile, confirm, drop=drop)
-    finally:
-        source.open_pool_connections()
-
-
 try:
     from mx.DateTime import DateTimeType, DateTimeDeltaType
 except ImportError:
@@ -196,25 +163,12 @@ class SQLAdapterMixIn(object):
         #self.dbapi_module.type_code_test(cnx.cursor())
         return cnx
 
-    def backup_to_file(self, backupfile, confirm):
+    def backup_to_file(self, backupfile):
         cmd = self.dbhelper.backup_command(self.dbname, self.dbhost,
                                            self.dbuser, backupfile,
                                            keepownership=False)
-        backupdir = os.path.dirname(backupfile)
-        if not os.path.exists(backupdir):
-            if confirm('%s does not exist. Create it?' % backupdir,
-                       abort=False, shell=False):
-                os.makedirs(backupdir)
-            else:
-                print '-> failed to backup instance'
-                return
         if os.system(cmd):
-            print '-> error trying to backup with command', cmd
-            if not confirm('Continue anyway?', default='n'):
-                raise SystemExit(1)
-        else:
-            print '-> backup file',  backupfile
-            restrict_perms_to_user(backupfile, self.info)
+            raise Exception('Failed command: %s' % cmd)
 
     def restore_from_file(self, backupfile, confirm, drop=True):
         for cmd in self.dbhelper.restore_commands(self.dbname, self.dbhost,
@@ -222,19 +176,8 @@ class SQLAdapterMixIn(object):
                                                   self.encoding,
                                                   keepownership=False,
                                                   drop=drop):
-            while True:
-                print cmd
-                if os.system(cmd):
-                    print '-> error while restoring the base'
-                    answer = confirm('Continue anyway?',
-                                     shell=False, abort=False, retry=True)
-                    if not answer:
-                        raise SystemExit(1)
-                    if answer == 1: # 1: continue, 2: retry
-                        break
-                else:
-                    break
-        print '-> database restored.'
+            if os.system(cmd):
+                raise Exception('Failed command: %s' % cmd)
 
     def merge_args(self, args, query_args):
         if args is not None:

vregistry.py

@@ -125,6 +125,7 @@ class Registry(dict):
 
     # dynamic selection methods ################################################
 
+    @deprecated('use select instead of object_by_id')
     def object_by_id(self, oid, *args, **kwargs):
         """return object with the given oid. Only one object is expected to be
         found.
@@ -143,9 +144,9 @@ class Registry(dict):
         raise `ObjectNotFound` if not object with id <oid> in <registry>
         raise `NoSelectableObject` if not object apply
         """
-        return self.select_best(self[oid], *args, **kwargs)
+        return self._select_best(self[oid], *args, **kwargs)
 
-    def select_object(self, oid, *args, **kwargs):
+    def select_or_none(self, oid, *args, **kwargs):
         """return the most specific object among those with the given oid
         according to the given context, or None if no object applies.
         """
@@ -153,6 +154,8 @@ class Registry(dict):
             return self.select(oid, *args, **kwargs)
         except (NoSelectableObject, ObjectNotFound):
             return None
+    select_object = deprecated('use select_or_none instead of select_object'
+                               )(select_or_none)
 
     def possible_objects(self, *args, **kwargs):
         """return an iterator on possible objects in this registry for the given
@@ -160,11 +163,11 @@ class Registry(dict):
         """
         for appobjects in self.itervalues():
             try:
-                yield self.select_best(appobjects, *args, **kwargs)
+                yield self._select_best(appobjects, *args, **kwargs)
             except NoSelectableObject:
                 continue
 
-    def select_best(self, appobjects, *args, **kwargs):
+    def _select_best(self, appobjects, *args, **kwargs):
         """return an instance of the most specific object according
         to parameters
 
@@ -194,7 +197,7 @@ class Registry(dict):
                                    [repr(v) for v in winners]))
         # return the result of calling the appobject
         return winners[0](*args, **kwargs)
-
+    select_best = deprecated('select_best is now private')(_select_best)
 
 class VRegistry(dict):
     """class responsible to register, propose and select the various
@@ -240,12 +243,12 @@ class VRegistry(dict):
         """
         return self[registry].select(oid, *args, **kwargs)
 
-    @deprecated('use vreg[registry].select_object(oid, *args, **kwargs)')
+    @deprecated('use vreg[registry].select_or_none(oid, *args, **kwargs)')
     def select_object(self, registry, oid, *args, **kwargs):
         """return the most specific object in <registry>.<oid> according to
         the given context, or None if no object apply
         """
-        return self[registry].select_object(oid, *args, **kwargs)
+        return self[registry].select_or_none(oid, *args, **kwargs)
 
     @deprecated('use vreg[registry].possible_objects(*args, **kwargs)')
     def possible_objects(self, registry, *args, **kwargs):

web/controller.py

@@ -93,7 +93,7 @@ class Controller(AppObject):
             self.ensure_ro_rql(rql)
             if not isinstance(rql, unicode):
                 rql = unicode(rql, self.req.encoding)
-            pp = self.vreg['components'].select_object('magicsearch', self.req)
+            pp = self.vreg['components'].select_or_none('magicsearch', self.req)
             if pp is not None:
                 self.rset = pp.process_query(rql, self.req)
         return self.rset

web/facet.py

@@ -64,8 +64,8 @@ def filtered_variable(rqlst):
 
 
 def get_facet(req, facetid, rqlst, mainvar):
-    return req.vreg['facets'].object_by_id(facetid, req, rqlst=rqlst,
-                                           filtered_variable=mainvar)
+    return req.vreg['facets'].select(facetid, req, rqlst=rqlst,
+                                     filtered_variable=mainvar)
 
 
 def filter_hiddens(w, **kwargs):

web/test/unittest_views_navigation.py

@@ -40,10 +40,10 @@ class NavigationTC(EnvBasedTC):
     def test_navigation_selection_not_enough(self):
         req = self.request()
         rset = self.execute('Any X,N LIMIT 10 WHERE X name N')
-        navcomp = self.vreg['components'].select_object('navigation', req, rset=rset)
+        navcomp = self.vreg['components'].select_or_none('navigation', req, rset=rset)
         self.assertEquals(navcomp, None)
         req.set_search_state('W:X:Y:Z')
-        navcomp = self.vreg['components'].select_object('navigation', req, rset=rset)
+        navcomp = self.vreg['components'].select_or_none('navigation', req, rset=rset)
         self.assertEquals(navcomp, None)
         req.set_search_state('normal')
 

web/views/basetemplates.py

@@ -154,12 +154,11 @@ class TheMainTemplate(MainTemplate):
         w(u'<div id="page"><table width="100%" border="0" id="mainLayout"><tr>\n')
         self.nav_column(view, 'left')
         w(u'<td id="contentcol">\n')
-        rqlcomp = self.vreg['components'].select_object('rqlinput', self.req,
-                                                        rset=self.rset)
+        components = self.vreg['components']
+        rqlcomp = components.select_or_none('rqlinput', self.req, rset=self.rset)
         if rqlcomp:
             rqlcomp.render(w=self.w, view=view)
-        msgcomp = self.vreg['components'].select_object('applmessages',
-                                                        self.req, rset=self.rset)
+        msgcomp = components.select_or_none('applmessages', self.req, rset=self.rset)
         if msgcomp:
             msgcomp.render(w=self.w)
         self.content_header(view)
@@ -299,8 +298,8 @@ class HTMLHeader(View):
             self.req.add_js(jscript, localfile=False)
 
     def alternates(self):
-        urlgetter = self.vreg['components'].select_object('rss_feed_url',
-                                            self.req, rset=self.rset)
+        urlgetter = self.vreg['components'].select_or_none('rss_feed_url',
+                                                           self.req, rset=self.rset)
         if urlgetter is not None:
             self.whead(u'<link rel="alternate" type="application/rss+xml" title="RSS feed" href="%s"/>\n'
                        %  xml_escape(urlgetter.feed_url()))

web/views/navigation.py

@@ -149,8 +149,8 @@ class SortedNavigation(NavigationComponent):
 def limit_rset_using_paged_nav(self, req, rset, w, forcedisplay=False,
                                show_all_option=True, page_size=None):
     if not (forcedisplay or req.form.get('__force_display') is not None):
-        nav = self.vreg['components'].select_object('navigation', req,
-                                      rset=rset, page_size=page_size)
+        nav = self.vreg['components'].select_or_none('navigation', req,
+                                                     rset=rset, page_size=page_size)
         if nav:
             # get boundaries before component rendering
             start, stop = nav.page_boundaries()

web/views/urlpublishing.py

@@ -217,6 +217,7 @@ class ActionPathEvaluator(URLPathEvaluator):
             raise PathDontMatch()
         # remove last part and see if this is something like an actions
         # if so, call
+        # XXX bad smell: refactor to simpler code
         try:
             actionsreg = self.vreg['actions']
             requested = parts.pop(-1)
@@ -232,7 +233,7 @@ class ActionPathEvaluator(URLPathEvaluator):
                 continue
             else:
                 try:
-                    action = actionsreg.select_best(actions, req, rset=rset)
+                    action = actionsreg._select_best(actions, req, rset=rset)
                 except RegistryException:
                     continue
                 else:

wsgi/handler.py

@@ -97,7 +97,7 @@ class CubicWebWSGIApplication(object):
 #         assert self.base_url[-1] == '/'
 #         self.https_url = config['https-url']
 #         assert not self.https_url or self.https_url[-1] == '/'
-        self.url_rewriter = self.appli.vreg.select_object('components', 'urlrewriter')
+        self.url_rewriter = self.appli.vreg['components'].select_or_none('urlrewriter')
 
     def _render(self, req):
         """this function performs the actual rendering