storages.py 11 KB
Newer Older
1
# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
"""custom storages for the system source"""
19

20
import os
21
import sys
22
from os import unlink, path as osp
23
from contextlib import contextmanager
24

25
26
from yams.schema import role_name

27
from cubicweb import Binary, ValidationError
28
from cubicweb.server import hook
29
from cubicweb.server.edition import EditedEntity
30

31
32

def set_attribute_storage(repo, etype, attr, storage):
33
    repo.system_source.set_storage(etype, attr, storage)
34

35
def unset_attribute_storage(repo, etype, attr):
36
    repo.system_source.unset_storage(etype, attr)
37

38

39
class Storage(object):
40
41
42
    """abstract storage

    * If `source_callback` is true (by default), the callback will be run during
43
      query result process of fetched attribute's value and should have the
44
45
      following prototype::

46
        callback(self, source, session, value)
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

      where `value` is the value actually stored in the backend. None values
      will be skipped (eg callback won't be called).

    * if `source_callback` is false, the callback will be run during sql
      generation when some attribute with a custom storage is accessed and
      should have the following prototype::

        callback(self, generator, relation, linkedvar)

      where `generator` is the sql generator, `relation` the current rql syntax
      tree relation and linkedvar the principal syntax tree variable holding the
      attribute.
    """
    is_source_callback = True

    def callback(self, *args):
        """see docstring for prototype, which vary according to is_source_callback
65
66
67
68
69
70
71
72
73
74
75
76
        """
        raise NotImplementedError()

    def entity_added(self, entity, attr):
        """an entity using this storage for attr has been added"""
        raise NotImplementedError()
    def entity_updated(self, entity, attr):
        """an entity using this storage for attr has been updatded"""
        raise NotImplementedError()
    def entity_deleted(self, entity, attr):
        """an entity using this storage for attr has been deleted"""
        raise NotImplementedError()
77
78
79
    def migrate_entity(self, entity, attribute):
        """migrate an entity attribute to the storage"""
        raise NotImplementedError()
80
81
82
83

# TODO
# * make it configurable without code
# * better file path attribution
Sylvain Thénault's avatar
Sylvain Thénault committed
84
# * handle backup/restore
85

86
87
88
89
90
91
def uniquify_path(dirpath, basename):
    """return a unique file name for `basename` in `dirpath`, or None
    if all attemps failed.

    XXX subject to race condition.
    """
92
    path = osp.join(dirpath, basename.replace(osp.sep, '-'))
93
94
95
96
97
98
99
100
101
    if not osp.isfile(path):
        return path
    base, ext = osp.splitext(path)
    for i in xrange(1, 256):
        path = '%s%s%s' % (base, i, ext)
        if not osp.isfile(path):
            return path
    return None

102
103
104
105
106
107
108
109
110
111
112
@contextmanager
def fsimport(session):
    present = 'fs_importing' in session.transaction_data
    old_value = session.transaction_data.get('fs_importing')
    session.transaction_data['fs_importing'] = True
    yield
    if present:
        session.transaction_data['fs_importing'] = old_value
    else:
        del session.transaction_data['fs_importing']

Sylvain Thénault's avatar
Sylvain Thénault committed
113

114
115
class BytesFileSystemStorage(Storage):
    """store Bytes attribute value on the file system"""
116
    def __init__(self, defaultdir, fsencoding='utf-8', wmode=0444):
117
118
        if type(defaultdir) is unicode:
            defaultdir = defaultdir.encode(fsencoding)
119
        self.default_directory = defaultdir
120
        self.fsencoding = fsencoding
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
        # extra umask to use when creating file
        # 0444 as in "only allow read bit in permission"
        self._wmode = wmode

    def _writecontent(self, path, binary):
        """write the content of a binary in readonly file

        As the bfss never alter a create file it does not prevent it to work as
        intended. This is a beter safe than sorry approach.
        """
        write_flag = os.O_WRONLY | os.O_CREAT | os.O_EXCL
        if sys.platform == 'win32':
            write_flag |= os.O_BINARY
        fd = os.open(path, write_flag, self._wmode)
        fileobj = os.fdopen(fd, 'wb')
        binary.to_file(fileobj)
        fileobj.close()

139

140
    def callback(self, source, session, value):
141
142
143
        """sql generator callback when some attribute with a custom storage is
        accessed
        """
144
145
        fpath = source.binary_to_str(value)
        try:
146
            return Binary.from_file(fpath)
147
        except EnvironmentError as ex:
148
149
            source.critical("can't open %s: %s", value, ex)
            return None
150
151
152

    def entity_added(self, entity, attr):
        """an entity using this storage for attr has been added"""
153
        if entity._cw.transaction_data.get('fs_importing'):
154
            binary = Binary.from_file(entity.cw_edited[attr].getvalue())
155
            entity._cw_dont_cache_attribute(attr, repo_side=True)
156
        else:
157
            binary = entity.cw_edited.pop(attr)
158
159
            fpath = self.new_fs_path(entity, attr)
            # bytes storage used to store file's path
160
            entity.cw_edited.edited_attribute(attr, Binary(fpath))
161
            self._writecontent(fpath, binary)
162
            AddFileOp.get_instance(entity._cw).add_data(fpath)
163
        return binary
164
165

    def entity_updated(self, entity, attr):
166
        """an entity using this storage for attr has been updated"""
167
        # get the name of the previous file containing the value
168
        oldpath = self.current_fs_path(entity, attr)
169
        if entity._cw.transaction_data.get('fs_importing'):
170
171
172
            # If we are importing from the filesystem, the file already exists.
            # We do not need to create it but we need to fetch the content of
            # the file as the actual content of the attribute
173
            fpath = entity.cw_edited[attr].getvalue()
174
            entity._cw_dont_cache_attribute(attr, repo_side=True)
175
            assert fpath is not None
176
            binary = Binary.from_file(fpath)
177
        else:
178
179
180
181
182
183
184
185
186
            # We must store the content of the attributes
            # into a file to stay consistent with the behaviour of entity_add.
            # Moreover, the BytesFileSystemStorage expects to be able to
            # retrieve the current value of the attribute at anytime by reading
            # the file on disk. To be able to rollback things, use a new file
            # and keep the old one that will be removed on commit if everything
            # went ok.
            #
            # fetch the current attribute value in memory
187
            binary = entity.cw_edited.pop(attr)
188
189
190
191
192
193
194
            if binary is None:
                fpath = None
            else:
                # Get filename for it
                fpath = self.new_fs_path(entity, attr)
                assert not osp.exists(fpath)
                # write attribute value on disk
195
                self._writecontent(fpath, binary)
196
197
198
                # Mark the new file as added during the transaction.
                # The file will be removed on rollback
                AddFileOp.get_instance(entity._cw).add_data(fpath)
199
            # reinstall poped value
200
201
202
            if fpath is None:
                entity.cw_edited.edited_attribute(attr, None)
            else:
203
                # register the new location for the file.
204
                entity.cw_edited.edited_attribute(attr, Binary(fpath))
205
        if oldpath is not None and oldpath != fpath:
206
207
            # Mark the old file as useless so the file will be removed at
            # commit.
208
            DeleteFileOp.get_instance(entity._cw).add_data(oldpath)
209
        return binary
210
211
212

    def entity_deleted(self, entity, attr):
        """an entity using this storage for attr has been deleted"""
213
        fpath = self.current_fs_path(entity, attr)
214
215
        if fpath is not None:
            DeleteFileOp.get_instance(entity._cw).add_data(fpath)
216
217

    def new_fs_path(self, entity, attr):
218
219
220
221
222
        # We try to get some hint about how to name the file using attribute's
        # name metadata, so we use the real file name and extension when
        # available. Keeping the extension is useful for example in the case of
        # PIL processing that use filename extension to detect content-type, as
        # well as providing more understandable file names on the fs.
223
        basename = [str(entity.eid), attr]
224
        name = entity.cw_attr_metadata(attr, 'name')
225
        if name is not None:
226
            basename.append(name.encode(self.fsencoding))
227
228
        fspath = uniquify_path(self.default_directory,
                               '_'.join(basename))
229
230
        if fspath is None:
            msg = entity._cw._('failed to uniquify path (%s, %s)') % (
231
                self.default_directory, '_'.join(basename))
232
            raise ValidationError(entity.eid, {role_name(attr, 'subject'): msg})
233
234
235
        return fspath

    def current_fs_path(self, entity, attr):
236
237
238
        """return the current fs_path of the attribute, or None is the attr is
        not stored yet.
        """
239
        sysource = entity._cw.repo.system_source
240
241
        cu = sysource.doexec(entity._cw,
                             'SELECT cw_%s FROM cw_%s WHERE cw_eid=%s' % (
242
                             attr, entity.cw_etype, entity.eid))
243
244
        rawvalue = cu.fetchone()[0]
        if rawvalue is None: # no previous value
245
            return None
246
        return sysource._process_value(rawvalue, cu.description[0],
247
                                       binarywrap=str)
248

249
250
    def migrate_entity(self, entity, attribute):
        """migrate an entity attribute to the storage"""
251
        entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache)
252
253
254
255
        self.entity_added(entity, attribute)
        session = entity._cw
        source = session.repo.system_source
        attrs = source.preprocess_entity(entity)
256
        sql = source.sqlgen.update('cw_' + entity.cw_etype, attrs,
257
258
                                   ['cw_eid'])
        source.doexec(session, sql, attrs)
259
        entity.cw_edited = None
260

261

262
class AddFileOp(hook.DataOperationMixIn, hook.Operation):
263
    def rollback_event(self):
264
        for filepath in self.get_data():
265
266
            try:
                unlink(filepath)
267
            except Exception as ex:
268
                self.error('cant remove %s: %s' % (filepath, ex))
269

270
class DeleteFileOp(hook.DataOperationMixIn, hook.Operation):
271
    def postcommit_event(self):
272
        for filepath in self.get_data():
273
274
            try:
                unlink(filepath)
275
            except Exception as ex:
276
                self.error('cant remove %s: %s' % (filepath, ex))