storages.py 10.1 KB
Newer Older
1
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
"""custom storages for the system source"""
19

20
import os
21
from os import unlink, path as osp
22
from contextlib import contextmanager
23

24
25
from yams.schema import role_name

26
from cubicweb import Binary, ValidationError
27
from cubicweb.server import hook
28
from cubicweb.server.edition import EditedEntity
29

30
31

def set_attribute_storage(repo, etype, attr, storage):
32
    repo.system_source.set_storage(etype, attr, storage)
33

34
def unset_attribute_storage(repo, etype, attr):
35
    repo.system_source.unset_storage(etype, attr)
36

37

38
class Storage(object):
39
40
41
    """abstract storage

    * If `source_callback` is true (by default), the callback will be run during
42
      query result process of fetched attribute's value and should have the
43
44
      following prototype::

45
        callback(self, source, session, value)
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

      where `value` is the value actually stored in the backend. None values
      will be skipped (eg callback won't be called).

    * if `source_callback` is false, the callback will be run during sql
      generation when some attribute with a custom storage is accessed and
      should have the following prototype::

        callback(self, generator, relation, linkedvar)

      where `generator` is the sql generator, `relation` the current rql syntax
      tree relation and linkedvar the principal syntax tree variable holding the
      attribute.
    """
    is_source_callback = True

    def callback(self, *args):
        """see docstring for prototype, which vary according to is_source_callback
64
65
66
67
68
69
70
71
72
73
74
75
        """
        raise NotImplementedError()

    def entity_added(self, entity, attr):
        """an entity using this storage for attr has been added"""
        raise NotImplementedError()
    def entity_updated(self, entity, attr):
        """an entity using this storage for attr has been updatded"""
        raise NotImplementedError()
    def entity_deleted(self, entity, attr):
        """an entity using this storage for attr has been deleted"""
        raise NotImplementedError()
76
77
78
    def migrate_entity(self, entity, attribute):
        """migrate an entity attribute to the storage"""
        raise NotImplementedError()
79
80
81
82

# TODO
# * make it configurable without code
# * better file path attribution
Sylvain Thénault's avatar
Sylvain Thénault committed
83
# * handle backup/restore
84

85
86
87
88
89
90
def uniquify_path(dirpath, basename):
    """return a unique file name for `basename` in `dirpath`, or None
    if all attemps failed.

    XXX subject to race condition.
    """
91
    path = osp.join(dirpath, basename.replace(osp.sep, '-'))
92
93
94
95
96
97
98
99
100
    if not osp.isfile(path):
        return path
    base, ext = osp.splitext(path)
    for i in xrange(1, 256):
        path = '%s%s%s' % (base, i, ext)
        if not osp.isfile(path):
            return path
    return None

101
102
103
104
105
106
107
108
109
110
111
@contextmanager
def fsimport(session):
    present = 'fs_importing' in session.transaction_data
    old_value = session.transaction_data.get('fs_importing')
    session.transaction_data['fs_importing'] = True
    yield
    if present:
        session.transaction_data['fs_importing'] = old_value
    else:
        del session.transaction_data['fs_importing']

Sylvain Thénault's avatar
Sylvain Thénault committed
112

113
114
class BytesFileSystemStorage(Storage):
    """store Bytes attribute value on the file system"""
115
    def __init__(self, defaultdir, fsencoding='utf-8'):
116
117
        if type(defaultdir) is unicode:
            defaultdir = defaultdir.encode(fsencoding)
118
        self.default_directory = defaultdir
119
        self.fsencoding = fsencoding
120

121
    def callback(self, source, session, value):
122
123
124
        """sql generator callback when some attribute with a custom storage is
        accessed
        """
125
126
        fpath = source.binary_to_str(value)
        try:
127
            return Binary.from_file(fpath)
128
        except EnvironmentError, ex:
129
130
            source.critical("can't open %s: %s", value, ex)
            return None
131
132
133

    def entity_added(self, entity, attr):
        """an entity using this storage for attr has been added"""
134
        if entity._cw.transaction_data.get('fs_importing'):
135
            binary = Binary.from_file(entity.cw_edited[attr].getvalue())
136
        else:
137
            binary = entity.cw_edited.pop(attr)
138
139
            fpath = self.new_fs_path(entity, attr)
            # bytes storage used to store file's path
140
            entity.cw_edited.edited_attribute(attr, Binary(fpath))
141
            binary.to_file(fpath)
142
            AddFileOp.get_instance(entity._cw).add_data(fpath)
143
        return binary
144
145

    def entity_updated(self, entity, attr):
146
        """an entity using this storage for attr has been updated"""
147
        # get the name of the previous file containing the value
148
        oldpath = self.current_fs_path(entity, attr)
149
        if entity._cw.transaction_data.get('fs_importing'):
150
151
152
            # If we are importing from the filesystem, the file already exists.
            # We do not need to create it but we need to fetch the content of
            # the file as the actual content of the attribute
153
            fpath = entity.cw_edited[attr].getvalue()
154
            assert fpath is not None
155
            binary = Binary.from_file(fpath)
156
        else:
157
158
159
160
161
162
163
164
165
            # We must store the content of the attributes
            # into a file to stay consistent with the behaviour of entity_add.
            # Moreover, the BytesFileSystemStorage expects to be able to
            # retrieve the current value of the attribute at anytime by reading
            # the file on disk. To be able to rollback things, use a new file
            # and keep the old one that will be removed on commit if everything
            # went ok.
            #
            # fetch the current attribute value in memory
166
            binary = entity.cw_edited.pop(attr)
167
168
169
170
171
172
173
            if binary is None:
                fpath = None
            else:
                # Get filename for it
                fpath = self.new_fs_path(entity, attr)
                assert not osp.exists(fpath)
                # write attribute value on disk
174
                binary.to_file(fpath)
175
176
177
                # Mark the new file as added during the transaction.
                # The file will be removed on rollback
                AddFileOp.get_instance(entity._cw).add_data(fpath)
178
        if oldpath != fpath:
179
            # register the new location for the file.
180
181
182
183
            if fpath is None:
                entity.cw_edited.edited_attribute(attr, None)
            else:
                entity.cw_edited.edited_attribute(attr, Binary(fpath))
184
185
            # Mark the old file as useless so the file will be removed at
            # commit.
186
187
            if oldpath is not None:
                DeleteFileOp.get_instance(entity._cw).add_data(oldpath)
188
        return binary
189
190
191

    def entity_deleted(self, entity, attr):
        """an entity using this storage for attr has been deleted"""
192
        fpath = self.current_fs_path(entity, attr)
193
194
        if fpath is not None:
            DeleteFileOp.get_instance(entity._cw).add_data(fpath)
195
196

    def new_fs_path(self, entity, attr):
197
198
199
200
201
        # We try to get some hint about how to name the file using attribute's
        # name metadata, so we use the real file name and extension when
        # available. Keeping the extension is useful for example in the case of
        # PIL processing that use filename extension to detect content-type, as
        # well as providing more understandable file names on the fs.
202
        basename = [str(entity.eid), attr]
203
        name = entity.cw_attr_metadata(attr, 'name')
204
        if name is not None:
205
            basename.append(name.encode(self.fsencoding))
206
207
        fspath = uniquify_path(self.default_directory,
                               '_'.join(basename))
208
209
        if fspath is None:
            msg = entity._cw._('failed to uniquify path (%s, %s)') % (
210
                self.default_directory, '_'.join(basename))
211
            raise ValidationError(entity.eid, {role_name(attr, 'subject'): msg})
212
213
214
        return fspath

    def current_fs_path(self, entity, attr):
215
216
217
        """return the current fs_path of the attribute, or None is the attr is
        not stored yet.
        """
218
        sysource = entity._cw.cnxset.source('system')
219
220
        cu = sysource.doexec(entity._cw,
                             'SELECT cw_%s FROM cw_%s WHERE cw_eid=%s' % (
221
                             attr, entity.__regid__, entity.eid))
222
223
        rawvalue = cu.fetchone()[0]
        if rawvalue is None: # no previous value
224
            return None
225
        return sysource._process_value(rawvalue, cu.description[0],
226
                                       binarywrap=str)
227

228
229
    def migrate_entity(self, entity, attribute):
        """migrate an entity attribute to the storage"""
230
        entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache)
231
232
233
234
235
236
237
        self.entity_added(entity, attribute)
        session = entity._cw
        source = session.repo.system_source
        attrs = source.preprocess_entity(entity)
        sql = source.sqlgen.update('cw_' + entity.__regid__, attrs,
                                   ['cw_eid'])
        source.doexec(session, sql, attrs)
238
        entity.cw_edited = None
239

240

241
class AddFileOp(hook.DataOperationMixIn, hook.Operation):
242
    def rollback_event(self):
243
        for filepath in self.get_data():
244
245
246
247
            try:
                unlink(filepath)
            except Exception, ex:
                self.error('cant remove %s: %s' % (filepath, ex))
248

249
class DeleteFileOp(hook.DataOperationMixIn, hook.Operation):
250
    def postcommit_event(self):
251
        for filepath in self.get_data():
252
253
254
255
            try:
                unlink(filepath)
            except Exception, ex:
                self.error('cant remove %s: %s' % (filepath, ex))