storages.py 10.5 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
"""custom storages for the system source"""
19

20
import os
21
import sys
22
from os import unlink, path as osp
23
from contextlib import contextmanager
24
import tempfile
25

26
27
from yams.schema import role_name

28
from cubicweb import Binary, ValidationError
29
from cubicweb.server import hook
30
from cubicweb.server.edition import EditedEntity
31

32
33

def set_attribute_storage(repo, etype, attr, storage):
34
    repo.system_source.set_storage(etype, attr, storage)
35

36
def unset_attribute_storage(repo, etype, attr):
37
    repo.system_source.unset_storage(etype, attr)
38

39

40
class Storage(object):
41
42
43
    """abstract storage

    * If `source_callback` is true (by default), the callback will be run during
44
      query result process of fetched attribute's value and should have the
45
46
      following prototype::

47
        callback(self, source, cnx, value)
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

      where `value` is the value actually stored in the backend. None values
      will be skipped (eg callback won't be called).

    * if `source_callback` is false, the callback will be run during sql
      generation when some attribute with a custom storage is accessed and
      should have the following prototype::

        callback(self, generator, relation, linkedvar)

      where `generator` is the sql generator, `relation` the current rql syntax
      tree relation and linkedvar the principal syntax tree variable holding the
      attribute.
    """
    is_source_callback = True

    def callback(self, *args):
        """see docstring for prototype, which vary according to is_source_callback
66
67
68
69
70
71
72
73
74
75
76
77
        """
        raise NotImplementedError()

    def entity_added(self, entity, attr):
        """an entity using this storage for attr has been added"""
        raise NotImplementedError()
    def entity_updated(self, entity, attr):
        """an entity using this storage for attr has been updatded"""
        raise NotImplementedError()
    def entity_deleted(self, entity, attr):
        """an entity using this storage for attr has been deleted"""
        raise NotImplementedError()
78
79
80
    def migrate_entity(self, entity, attribute):
        """migrate an entity attribute to the storage"""
        raise NotImplementedError()
81
82
83
84

# TODO
# * make it configurable without code
# * better file path attribution
Sylvain Thénault's avatar
Sylvain Thénault committed
85
# * handle backup/restore
86

87
def uniquify_path(dirpath, basename):
88
    """return a file descriptor and unique file name for `basename` in `dirpath`
89
    """
90
    path = basename.replace(osp.sep, '-')
91
    base, ext = osp.splitext(path)
92
    return tempfile.mkstemp(prefix=base, suffix=ext, dir=dirpath)
93

94
@contextmanager
95
96
97
98
def fsimport(cnx):
    present = 'fs_importing' in cnx.transaction_data
    old_value = cnx.transaction_data.get('fs_importing')
    cnx.transaction_data['fs_importing'] = True
99
100
    yield
    if present:
101
        cnx.transaction_data['fs_importing'] = old_value
102
    else:
103
        del cnx.transaction_data['fs_importing']
104

Sylvain Thénault's avatar
Sylvain Thénault committed
105

106
107
class BytesFileSystemStorage(Storage):
    """store Bytes attribute value on the file system"""
Samuel Trégouët's avatar
Samuel Trégouët committed
108
    def __init__(self, defaultdir, fsencoding='utf-8', wmode=0o444):
109
110
        if type(defaultdir) is unicode:
            defaultdir = defaultdir.encode(fsencoding)
111
        self.default_directory = defaultdir
112
        self.fsencoding = fsencoding
113
114
115
116
        # extra umask to use when creating file
        # 0444 as in "only allow read bit in permission"
        self._wmode = wmode

117
    def _writecontent(self, fd, binary):
118
119
        """write the content of a binary in readonly file

120
121
        As the bfss never alters an existing file it does not prevent it from
        working as intended. This is a better safe than sorry approach.
122
        """
123
        os.fchmod(fd, self._wmode)
124
125
126
127
        fileobj = os.fdopen(fd, 'wb')
        binary.to_file(fileobj)
        fileobj.close()

128

129
    def callback(self, source, cnx, value):
130
131
132
        """sql generator callback when some attribute with a custom storage is
        accessed
        """
133
134
        fpath = source.binary_to_str(value)
        try:
135
            return Binary.from_file(fpath)
136
        except EnvironmentError as ex:
137
138
            source.critical("can't open %s: %s", value, ex)
            return None
139
140
141

    def entity_added(self, entity, attr):
        """an entity using this storage for attr has been added"""
142
        if entity._cw.transaction_data.get('fs_importing'):
143
            binary = Binary.from_file(entity.cw_edited[attr].getvalue())
144
        else:
145
            binary = entity.cw_edited.pop(attr)
146
            fd, fpath = self.new_fs_path(entity, attr)
147
            # bytes storage used to store file's path
148
            entity.cw_edited.edited_attribute(attr, Binary(fpath))
149
            self._writecontent(fd, binary)
150
            AddFileOp.get_instance(entity._cw).add_data(fpath)
151
        return binary
152
153

    def entity_updated(self, entity, attr):
154
        """an entity using this storage for attr has been updated"""
155
        # get the name of the previous file containing the value
156
        oldpath = self.current_fs_path(entity, attr)
157
        if entity._cw.transaction_data.get('fs_importing'):
158
159
160
            # If we are importing from the filesystem, the file already exists.
            # We do not need to create it but we need to fetch the content of
            # the file as the actual content of the attribute
161
            fpath = entity.cw_edited[attr].getvalue()
162
            assert fpath is not None
163
            binary = Binary.from_file(fpath)
164
        else:
165
166
167
168
169
170
171
172
173
            # We must store the content of the attributes
            # into a file to stay consistent with the behaviour of entity_add.
            # Moreover, the BytesFileSystemStorage expects to be able to
            # retrieve the current value of the attribute at anytime by reading
            # the file on disk. To be able to rollback things, use a new file
            # and keep the old one that will be removed on commit if everything
            # went ok.
            #
            # fetch the current attribute value in memory
174
            binary = entity.cw_edited.pop(attr)
175
176
177
178
            if binary is None:
                fpath = None
            else:
                # Get filename for it
179
                fd, fpath = self.new_fs_path(entity, attr)
180
                # write attribute value on disk
181
                self._writecontent(fd, binary)
182
183
184
                # Mark the new file as added during the transaction.
                # The file will be removed on rollback
                AddFileOp.get_instance(entity._cw).add_data(fpath)
185
            # reinstall poped value
186
187
188
            if fpath is None:
                entity.cw_edited.edited_attribute(attr, None)
            else:
189
                # register the new location for the file.
190
                entity.cw_edited.edited_attribute(attr, Binary(fpath))
191
        if oldpath is not None and oldpath != fpath:
192
193
            # Mark the old file as useless so the file will be removed at
            # commit.
194
            DeleteFileOp.get_instance(entity._cw).add_data(oldpath)
195
        return binary
196
197
198

    def entity_deleted(self, entity, attr):
        """an entity using this storage for attr has been deleted"""
199
        fpath = self.current_fs_path(entity, attr)
200
201
        if fpath is not None:
            DeleteFileOp.get_instance(entity._cw).add_data(fpath)
202
203

    def new_fs_path(self, entity, attr):
204
205
206
207
208
        # We try to get some hint about how to name the file using attribute's
        # name metadata, so we use the real file name and extension when
        # available. Keeping the extension is useful for example in the case of
        # PIL processing that use filename extension to detect content-type, as
        # well as providing more understandable file names on the fs.
209
        basename = [str(entity.eid), attr]
210
        name = entity.cw_attr_metadata(attr, 'name')
211
        if name is not None:
212
            basename.append(name.encode(self.fsencoding))
213
        fd, fspath = uniquify_path(self.default_directory,
214
                               '_'.join(basename))
215
216
        if fspath is None:
            msg = entity._cw._('failed to uniquify path (%s, %s)') % (
217
                self.default_directory, '_'.join(basename))
218
            raise ValidationError(entity.eid, {role_name(attr, 'subject'): msg})
219
        return fd, fspath
220
221

    def current_fs_path(self, entity, attr):
222
223
224
        """return the current fs_path of the attribute, or None is the attr is
        not stored yet.
        """
225
        sysource = entity._cw.repo.system_source
226
227
        cu = sysource.doexec(entity._cw,
                             'SELECT cw_%s FROM cw_%s WHERE cw_eid=%s' % (
228
                             attr, entity.cw_etype, entity.eid))
229
230
        rawvalue = cu.fetchone()[0]
        if rawvalue is None: # no previous value
231
            return None
232
        return sysource._process_value(rawvalue, cu.description[0],
233
                                       binarywrap=str)
234

235
236
    def migrate_entity(self, entity, attribute):
        """migrate an entity attribute to the storage"""
237
        entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache)
238
        self.entity_added(entity, attribute)
239
240
        cnx = entity._cw
        source = cnx.repo.system_source
241
        attrs = source.preprocess_entity(entity)
242
        sql = source.sqlgen.update('cw_' + entity.cw_etype, attrs,
243
                                   ['cw_eid'])
244
        source.doexec(cnx, sql, attrs)
245
        entity.cw_edited = None
246

247

248
class AddFileOp(hook.DataOperationMixIn, hook.Operation):
249
    def rollback_event(self):
250
        for filepath in self.get_data():
251
252
            try:
                unlink(filepath)
253
            except Exception as ex:
254
                self.error('cant remove %s: %s' % (filepath, ex))
255

256
class DeleteFileOp(hook.DataOperationMixIn, hook.Operation):
257
    def postcommit_event(self):
258
        for filepath in self.get_data():
259
260
            try:
                unlink(filepath)
261
            except Exception as ex:
262
                self.error('cant remove %s: %s' % (filepath, ex))