Commit 39de18bd authored by Katia Saurfelt's avatar Katia Saurfelt
Browse files

server/web api for accessing to deleted_entites

parent 9767cc516b4f
......@@ -7,7 +7,7 @@ software
distname = "cubicweb"
modname = "cubicweb"
numversion = (3, 6, 2)
numversion = (3, 7, 0)
version = '.'.join(str(num) for num in numversion)
license = 'LGPL'
......
......@@ -559,6 +559,8 @@ class NoHookRQLObjectStore(RQLObjectStore):
self._nb_inserted_types = 0
self._nb_inserted_relations = 0
self.rql = session.unsafe_execute
# disable undoing
session.undo_actions = frozenset()
def create_entity(self, etype, **kwargs):
for k, v in kwargs.iteritems():
......
......@@ -57,6 +57,7 @@ def multiple_connections_unfix():
etypescls = cwvreg.VRegistry.REGISTRY_FACTORY['etypes']
etypescls.etype_class = etypescls.orig_etype_class
class ConnectionProperties(object):
def __init__(self, cnxtype=None, lang=None, close=True, log=False):
self.cnxtype = cnxtype or 'pyro'
......@@ -546,16 +547,16 @@ class Connection(object):
self._closed = 1
def commit(self):
"""Commit any pending transaction to the database. Note that if the
database supports an auto-commit feature, this must be initially off. An
interface method may be provided to turn it back on.
"""Commit pending transaction for this connection to the repository.
may raises `Unauthorized` or `ValidationError` if we attempted to do
something we're not allowed to for security or integrity reason.
Database modules that do not support transactions should implement this
method with void functionality.
If the transaction is undoable, a transaction id will be returned.
"""
if not self._closed is None:
raise ProgrammingError('Connection is already closed')
self._repo.commit(self.sessionid)
return self._repo.commit(self.sessionid)
def rollback(self):
"""This method is optional since not all databases provide transaction
......@@ -582,6 +583,73 @@ class Connection(object):
req = self.request()
return self.cursor_class(self, self._repo, req=req)
# undo support ############################################################
def undoable_transactions(self, ueid=None, req=None, **actionfilters):
"""Return a list of undoable transaction objects by the connection's
user, ordered by descendant transaction time.
Managers may filter according to user (eid) who has done the transaction
using the `ueid` argument. Others will only see their own transactions.
Additional filtering capabilities is provided by using the following
named arguments:
* `etype` to get only transactions creating/updating/deleting entities
of the given type
* `eid` to get only transactions applied to entity of the given eid
* `action` to get only transactions doing the given action (action in
'C', 'U', 'D', 'A', 'R'). If `etype`, action can only be 'C', 'U' or
'D'.
* `public`: when additional filtering is provided, their are by default
only searched in 'public' actions, unless a `public` argument is given
and set to false.
"""
txinfos = self._repo.undoable_transactions(self.sessionid, ueid,
**actionfilters)
if req is None:
req = self.request()
for txinfo in txinfos:
txinfo.req = req
return txinfos
def transaction_info(self, txuuid, req=None):
"""Return transaction object for the given uid.
raise `NoSuchTransaction` if not found or if session's user is not
allowed (eg not in managers group and the transaction doesn't belong to
him).
"""
txinfo = self._repo.transaction_info(self.sessionid, txuuid)
if req is None:
req = self.request()
txinfo.req = req
return txinfo
def transaction_actions(self, txuuid, public=True):
"""Return an ordered list of action effectued during that transaction.
If public is true, return only 'public' actions, eg not ones triggered
under the cover by hooks, else return all actions.
raise `NoSuchTransaction` if the transaction is not found or if
session's user is not allowed (eg not in managers group and the
transaction doesn't belong to him).
"""
return self._repo.transaction_actions(self.sessionid, txuuid, public)
def undo_transaction(self, txuuid):
"""Undo the given transaction. Return potential restoration errors.
raise `NoSuchTransaction` if not found or if session's user is not
allowed (eg not in managers group and the transaction doesn't belong to
him).
"""
return self._repo.undo_transaction(self.sessionid, txuuid)
# cursor object ###############################################################
......
......@@ -106,8 +106,6 @@ class TestServerConfiguration(ServerConfiguration):
self.init_log(log_threshold, force=True)
# need this, usually triggered by cubicweb-ctl
self.load_cwctl_plugins()
self.global_set_option('anonymous-user', 'anon')
self.global_set_option('anonymous-password', 'anon')
anonymous_user = TwistedConfiguration.anonymous_user.im_func
......@@ -123,6 +121,8 @@ class TestServerConfiguration(ServerConfiguration):
super(TestServerConfiguration, self).load_configuration()
self.global_set_option('anonymous-user', 'anon')
self.global_set_option('anonymous-password', 'anon')
# no undo support in tests
self.global_set_option('undo-support', '')
def main_config_file(self):
"""return instance's control configuration file"""
......
......@@ -321,7 +321,10 @@ class CubicWebTC(TestCase):
@nocoverage
def commit(self):
self.cnx.commit()
try:
return self.cnx.commit()
finally:
self.session.set_pool() # ensure pool still set after commit
@nocoverage
def rollback(self):
......@@ -329,6 +332,8 @@ class CubicWebTC(TestCase):
self.cnx.rollback()
except ProgrammingError:
pass
finally:
self.session.set_pool() # ensure pool still set after commit
# # server side db api #######################################################
......
......@@ -461,7 +461,7 @@ class Entity(AppObject, dict):
all(matching_groups(e.get_groups('read')) for e in targets):
yield rschema, 'subject'
def to_complete_attributes(self, skip_bytes=True):
def to_complete_attributes(self, skip_bytes=True, skip_pwd=True):
for rschema, attrschema in self.e_schema.attribute_definitions():
# skip binary data by default
if skip_bytes and attrschema.type == 'Bytes':
......@@ -472,13 +472,13 @@ class Entity(AppObject, dict):
# password retreival is blocked at the repository server level
rdef = rschema.rdef(self.e_schema, attrschema)
if not self._cw.user.matching_groups(rdef.get_groups('read')) \
or attrschema.type == 'Password':
or (attrschema.type == 'Password' and skip_pwd):
self[attr] = None
continue
yield attr
_cw_completed = False
def complete(self, attributes=None, skip_bytes=True):
def complete(self, attributes=None, skip_bytes=True, skip_pwd=True):
"""complete this entity by adding missing attributes (i.e. query the
repository to fill the entity)
......@@ -495,7 +495,7 @@ class Entity(AppObject, dict):
V = varmaker.next()
rql = ['WHERE %s eid %%(x)s' % V]
selected = []
for attr in (attributes or self.to_complete_attributes(skip_bytes)):
for attr in (attributes or self.to_complete_attributes(skip_bytes, skip_pwd)):
# if attribute already in entity, nothing to do
if self.has_key(attr):
continue
......
......@@ -255,10 +255,11 @@ class GAESource(AbstractSource):
if asession.user.eid == entity.eid:
asession.user.update(dict(gaeentity))
def delete_entity(self, session, etype, eid):
def delete_entity(self, session, entity):
"""delete an entity from the source"""
# do not delay delete_entity as other modifications to ensure
# consistency
eid = entity.eid
key = Key(eid)
Delete(key)
session.clear_datastore_cache(key)
......
"""core hooks"""
"""core hooks
:organization: Logilab
:copyright: 2009-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses
"""
__docformat__ = "restructuredtext en"
from datetime import timedelta, datetime
from cubicweb.server import hook
class ServerStartupHook(hook.Hook):
"""task to cleanup expirated auth cookie entities"""
__regid__ = 'cw_cleanup_transactions'
events = ('server_startup',)
def __call__(self):
# XXX use named args and inner functions to avoid referencing globals
# which may cause reloading pb
lifetime = timedelta(days=self.repo.config['keep-transaction-lifetime'])
def cleanup_old_transactions(repo=self.repo, lifetime=lifetime):
mindate = datetime.now() - lifetime
session = repo.internal_session()
try:
session.system_sql(
'DELETE FROM transaction WHERE tx_time < %(time)s',
{'time': mindate})
# cleanup deleted entities
session.system_sql(
'DELETE FROM deleted_entities WHERE dtime < %(time)s',
{'time': mindate})
session.commit()
finally:
session.close()
self.repo.looping_task(60*60*24, cleanup_old_transactions, self.repo)
typemap = repo.system_source.dbhelper.TYPE_MAPPING
sqls = """
CREATE TABLE transactions (
tx_uuid CHAR(32) PRIMARY KEY NOT NULL,
tx_user INTEGER NOT NULL,
tx_time %s NOT NULL
);;
CREATE INDEX transactions_tx_user_idx ON transactions(tx_user);;
CREATE TABLE tx_entity_actions (
tx_uuid CHAR(32) REFERENCES transactions(tx_uuid) ON DELETE CASCADE,
txa_action CHAR(1) NOT NULL,
txa_public %s NOT NULL,
txa_order INTEGER,
eid INTEGER NOT NULL,
etype VARCHAR(64) NOT NULL,
changes %s
);;
CREATE INDEX tx_entity_actions_txa_action_idx ON tx_entity_actions(txa_action);;
CREATE INDEX tx_entity_actions_txa_public_idx ON tx_entity_actions(txa_public);;
CREATE INDEX tx_entity_actions_eid_idx ON tx_entity_actions(eid);;
CREATE INDEX tx_entity_actions_etype_idx ON tx_entity_actions(etype);;
CREATE TABLE tx_relation_actions (
tx_uuid CHAR(32) REFERENCES transactions(tx_uuid) ON DELETE CASCADE,
txa_action CHAR(1) NOT NULL,
txa_public %s NOT NULL,
txa_order INTEGER,
eid_from INTEGER NOT NULL,
eid_to INTEGER NOT NULL,
rtype VARCHAR(256) NOT NULL
);;
CREATE INDEX tx_relation_actions_txa_action_idx ON tx_relation_actions(txa_action);;
CREATE INDEX tx_relation_actions_txa_public_idx ON tx_relation_actions(txa_public);;
CREATE INDEX tx_relation_actions_eid_from_idx ON tx_relation_actions(eid_from);;
CREATE INDEX tx_relation_actions_eid_to_idx ON tx_relation_actions(eid_to)
""" % (typemap['Datetime'],
typemap['Boolean'], typemap['Bytes'], typemap['Boolean'])
for statement in sqls.split(';;'):
sql(statement)
......@@ -34,14 +34,15 @@ from cubicweb import ETYPE_NAME_MAP, ValidationError, Unauthorized
PURE_VIRTUAL_RTYPES = set(('identity', 'has_text',))
VIRTUAL_RTYPES = set(('eid', 'identity', 'has_text',))
# set of meta-relations available for every entity types
# set of meta-relations available for every entity types
META_RTYPES = set((
'owned_by', 'created_by', 'is', 'is_instance_of', 'identity',
'eid', 'creation_date', 'modification_date', 'has_text', 'cwuri',
))
SYSTEM_RTYPES = set(('require_permission', 'custom_workflow', 'in_state', 'wf_info_for'))
SYSTEM_RTYPES = set(('require_permission', 'custom_workflow', 'in_state',
'wf_info_for'))
# set of entity and relation types used to build the schema
# set of entity and relation types used to build the schema
SCHEMA_TYPES = set((
'CWEType', 'CWRType', 'CWAttribute', 'CWRelation',
'CWConstraint', 'CWConstraintType', 'RQLExpression',
......
......@@ -150,7 +150,7 @@ def init_repository(config, interactive=True, drop=False, vreg=None):
schemasql = sqlschema(schema, driver)
#skip_entities=[str(e) for e in schema.entities()
# if not repo.system_source.support_entity(str(e))])
sqlexec(schemasql, execute, pbtitle=_title)
sqlexec(schemasql, execute, pbtitle=_title, delimiter=';;')
sqlcursor.close()
sqlcnx.commit()
sqlcnx.close()
......
......@@ -24,9 +24,11 @@ import Queue
from os.path import join
from datetime import datetime
from time import time, localtime, strftime
#from pickle import dumps
from logilab.common.decorators import cached
from logilab.common.compat import any
from logilab.common import flatten
from yams import BadSchemaDefinition
from rql import RQLSyntaxError
......@@ -630,7 +632,7 @@ class Repository(object):
"""commit transaction for the session with the given id"""
self.debug('begin commit for session %s', sessionid)
try:
self._get_session(sessionid).commit()
return self._get_session(sessionid).commit()
except (ValidationError, Unauthorized):
raise
except:
......@@ -679,10 +681,42 @@ class Repository(object):
custom properties)
"""
session = self._get_session(sessionid, setpool=False)
# update session properties
for prop, value in props.items():
session.change_property(prop, value)
def undoable_transactions(self, sessionid, ueid=None, **actionfilters):
"""See :class:`cubicweb.dbapi.Connection.undoable_transactions`"""
session = self._get_session(sessionid, setpool=True)
try:
return self.system_source.undoable_transactions(session, ueid,
**actionfilters)
finally:
session.reset_pool()
def transaction_info(self, sessionid, txuuid):
"""See :class:`cubicweb.dbapi.Connection.transaction_info`"""
session = self._get_session(sessionid, setpool=True)
try:
return self.system_source.tx_info(session, txuuid)
finally:
session.reset_pool()
def transaction_actions(self, sessionid, txuuid, public=True):
"""See :class:`cubicweb.dbapi.Connection.transaction_actions`"""
session = self._get_session(sessionid, setpool=True)
try:
return self.system_source.tx_actions(session, txuuid, public)
finally:
session.reset_pool()
def undo_transaction(self, sessionid, txuuid):
"""See :class:`cubicweb.dbapi.Connection.undo_transaction`"""
session = self._get_session(sessionid, setpool=True)
try:
return self.system_source.undo_transaction(session, txuuid)
finally:
session.reset_pool()
# public (inter-repository) interface #####################################
def entities_modified_since(self, etypes, mtime):
......@@ -886,60 +920,58 @@ class Repository(object):
self.system_source.add_info(session, entity, source, extid, complete)
CleanupEidTypeCacheOp(session)
def delete_info(self, session, eid):
self._prepare_delete_info(session, eid)
self._delete_info(session, eid)
def delete_info(self, session, entity, sourceuri, extid):
"""called by external source when some entity known by the system source
has been deleted in the external source
"""
self._prepare_delete_info(session, entity, sourceuri)
self._delete_info(session, entity, sourceuri, extid)
def _prepare_delete_info(self, session, eid):
def _prepare_delete_info(self, session, entity, sourceuri):
"""prepare the repository for deletion of an entity:
* update the fti
* mark eid as being deleted in session info
* setup cache update operation
* if undoable, get back all entity's attributes and relation
"""
eid = entity.eid
self.system_source.fti_unindex_entity(session, eid)
pending = session.transaction_data.setdefault('pendingeids', set())
pending.add(eid)
CleanupEidTypeCacheOp(session)
def _delete_info(self, session, eid):
def _delete_info(self, session, entity, sourceuri, extid):
# attributes=None, relations=None):
"""delete system information on deletion of an entity:
* delete all relations on this entity
* transfer record from the entities table to the deleted_entities table
"""
etype, uri, extid = self.type_and_source_from_eid(eid, session)
self._clear_eid_relations(session, etype, eid)
self.system_source.delete_info(session, eid, etype, uri, extid)
def _clear_eid_relations(self, session, etype, eid):
"""when a entity is deleted, build and execute rql query to delete all
its relations
* delete all remaining relations from/to this entity
* call delete info on the system source which will transfer record from
the entities table to the deleted_entities table
"""
rql = []
eschema = self.schema.eschema(etype)
pendingrtypes = session.transaction_data.get('pendingrtypes', ())
# delete remaining relations: if user can delete the entity, he can
# delete all its relations without security checking
with security_enabled(session, read=False, write=False):
for rschema, targetschemas, x in eschema.relation_definitions():
eid = entity.eid
for rschema, _, role in entity.e_schema.relation_definitions():
rtype = rschema.type
if rtype in schema.VIRTUAL_RTYPES or rtype in pendingrtypes:
continue
var = '%s%s' % (rtype.upper(), x.upper())
if x == 'subject':
if role == 'subject':
# don't skip inlined relation so they are regularly
# deleted and so hooks are correctly called
selection = 'X %s %s' % (rtype, var)
selection = 'X %s Y' % rtype
else:
selection = '%s %s X' % (var, rtype)
selection = 'Y %s X' % rtype
rql = 'DELETE %s WHERE X eid %%(x)s' % selection
# if user can delete the entity, he can delete all its relations
# without security checking
session.execute(rql, {'x': eid}, 'x', build_descr=False)
self.system_source.delete_info(session, entity, sourceuri, extid)
def locate_relation_source(self, session, subject, rtype, object):
subjsource = self.source_from_eid(subject, session)
objsource = self.source_from_eid(object, session)
if not subjsource is objsource:
source = self.system_source
if not (subjsource.may_cross_relation(rtype)
if not (subjsource.may_cross_relation(rtype)
and objsource.may_cross_relation(rtype)):
raise MultiSourcesError(
"relation %s can't be crossed among sources"
......@@ -983,7 +1015,7 @@ class Repository(object):
self.hm.call_hooks('before_add_entity', session, entity=entity)
# XXX use entity.keys here since edited_attributes is not updated for
# inline relations
for attr in entity.keys():
for attr in entity.iterkeys():
rschema = eschema.subjrels[attr]
if not rschema.final: # inlined relation
relations.append((attr, entity[attr]))
......@@ -1094,19 +1126,16 @@ class Repository(object):
def glob_delete_entity(self, session, eid):
"""delete an entity and all related entities from the repository"""
# call delete_info before hooks
self._prepare_delete_info(session, eid)
etype, uri, extid = self.type_and_source_from_eid(eid, session)
entity = session.entity_from_eid(eid)
etype, sourceuri, extid = self.type_and_source_from_eid(eid, session)
self._prepare_delete_info(session, entity, sourceuri)
if server.DEBUG & server.DBG_REPO:
print 'DELETE entity', etype, eid
if eid == 937:
server.DEBUG |= (server.DBG_SQL | server.DBG_RQL | server.DBG_MORE)
source = self.sources_by_uri[uri]
source = self.sources_by_uri[sourceuri]
if source.should_call_hooks:
entity = session.entity_from_eid(eid)
self.hm.call_hooks('before_delete_entity', session, entity=entity)
self._delete_info(session, eid)
source.delete_entity(session, etype, eid)
self._delete_info(session, entity, sourceuri, extid)
source.delete_entity(session, entity)
if source.should_call_hooks:
self.hm.call_hooks('after_delete_entity', session, entity=entity)
# don't clear cache here this is done in a hook on commit
......
......@@ -127,6 +127,20 @@ connections will have this number of opened connections.',
'help': 'size of the parsed rql cache size.',
'group': 'main', 'inputlevel': 1,
}),
('undo-support',
{'type' : 'string', 'default': '',
'help': 'string defining actions that will have undo support: \
[C]reate [U]pdate [D]elete entities / [A]dd [R]emove relation. Leave it empty \
for no undo support, set it to CUDAR for full undo support, or to DR for \
support undoing of deletion only.',
'group': 'main', 'inputlevel': 1,
}),
('keep-transaction-lifetime',
{'type' : 'int', 'default': 7,
'help': 'number of days during which transaction records should be \
kept (hence undoable).',
'group': 'main', 'inputlevel': 1,
}),
('delay-full-text-indexation',
{'type' : 'yn', 'default': False,
'help': 'When full text indexation of entity has a too important cost'
......
......@@ -12,12 +12,13 @@ __docformat__ = "restructuredtext en"
import sys
import threading
from time import time
from uuid import uuid4
from logilab.common.deprecation import deprecated
from rql.nodes import VariableRef, Function, ETYPE_PYOBJ_MAP, etype_from_pyobj
from yams import BASE_TYPES
from cubicweb import Binary, UnknownEid
from cubicweb import Binary, UnknownEid, schema
from cubicweb.req import RequestSessionBase
from cubicweb.dbapi import ConnectionProperties
from cubicweb.utils import make_uid
......@@ -25,6 +26,10 @@ from cubicweb.rqlrewrite import RQLRewriter
ETYPE_PYOBJ_MAP[Binary] = 'Bytes'
NO_UNDO_TYPES = schema.SCHEMA_TYPES.copy()
NO_UNDO_TYPES.add('CWCache')
# XXX rememberme,forgotpwd,apycot,vcsfile
def is_final(rqlst, variable, args):
# try to find if this is a final var or not
for select in rqlst.children:
......@@ -110,6 +115,7 @@ class Session(RequestSessionBase):
"""tie session id, user, connections pool and other session data all
together
"""
is_internal_session = False
def __init__(self, user, repo, cnxprops=None, _id=None):
super(Session, self).__init__(repo.vreg)
......@@ -120,8 +126,14 @@ class Session(RequestSessionBase):
self.cnxtype = cnxprops.cnxtype
self.creation = time()
self.timestamp = self.creation
self.is_internal_session = False
self.default_mode = 'read'
# support undo for Create Update Delete entity / Add Remove relation
if repo.config.creating or repo.config.repairing or self.is_internal_session:
self.undo_actions = ()
else:
self.undo_actions = set(repo.config['undo-support'].upper())
if self.undo_actions - set('CUDAR'):
raise Exception('bad undo-support string in configuration')
# short cut to querier .execute method
self._execute = repo.querier.execute
# shared data, used to communicate extra information between the client
......@@ -334,7 +346,10 @@ class Session(RequestSessionBase):
# so we can't rely on simply checking session.read_security, but
# recalling the first transition from DEFAULT_SECURITY to something
# else (False actually) is not perfect but should be enough
self._threaddata.dbapi_query = oldmode is self.DEFAULT_SECURITY
#
# also reset dbapi_query to true when we go back to DEFAULT_SECURITY
self._threaddata.dbapi_query = (oldmode is self.DEFAULT_SECURITY
or activated is self.DEFAULT_SECURITY)
return oldmode
@property
......@@ -689,6 +704,7 @@ class Session(RequestSessionBase):
self.critical('error while %sing', trstate,
exc_info=sys.exc_info())
self.info('%s session %s done', trstate, self.id)
return self.transaction_uuid(set=False)
finally:
self._clear_thread_data()
self._touch()
......@@ -769,6 +785,27 @@ class Session(RequestSessionBase):
else:
self.pending_operations.insert(index, operation)
# undo support ############################################################
def undoable_action(self, action, ertype):
return action in self.undo_actions and not ertype in NO_UNDO_TYPES
# XXX elif transaction on mark it partial
def transaction_uuid(self, set=True):
try:
return self.transaction_data['tx_uuid']
except KeyError:
if not set:
return
self.transaction_data['tx_uuid'] = uuid = uuid4().hex
self.repo.system_source.start_undoable_transaction(self, uuid)
return uuid
def transaction_inc_action_counter(self):
num = self.transaction_data.setdefault('tx_action_count', 0) + 1
self.transaction_data['tx_action_count'] = num