storages.py 7.21 KB
Newer Older
1
2
3
"""custom storages for the system source"""
from os import unlink, path as osp

4
5
from yams.schema import role_name

6
from cubicweb import Binary
7
from cubicweb.server import hook
8
9

def set_attribute_storage(repo, etype, attr, storage):
10
    repo.system_source.set_storage(etype, attr, storage)
11

12
def unset_attribute_storage(repo, etype, attr):
13
    repo.system_source.unset_storage(etype, attr)
14
15

class Storage(object):
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
    """abstract storage

    * If `source_callback` is true (by default), the callback will be run during
      query result process of fetched attribute's valu and should have the
      following prototype::

        callback(self, source, value)

      where `value` is the value actually stored in the backend. None values
      will be skipped (eg callback won't be called).

    * if `source_callback` is false, the callback will be run during sql
      generation when some attribute with a custom storage is accessed and
      should have the following prototype::

        callback(self, generator, relation, linkedvar)

      where `generator` is the sql generator, `relation` the current rql syntax
      tree relation and linkedvar the principal syntax tree variable holding the
      attribute.
    """
    is_source_callback = True

    def callback(self, *args):
        """see docstring for prototype, which vary according to is_source_callback
41
42
43
44
45
46
47
48
49
50
51
52
        """
        raise NotImplementedError()

    def entity_added(self, entity, attr):
        """an entity using this storage for attr has been added"""
        raise NotImplementedError()
    def entity_updated(self, entity, attr):
        """an entity using this storage for attr has been updatded"""
        raise NotImplementedError()
    def entity_deleted(self, entity, attr):
        """an entity using this storage for attr has been deleted"""
        raise NotImplementedError()
53
54
55
    def migrate_entity(self, entity, attribute):
        """migrate an entity attribute to the storage"""
        raise NotImplementedError()
56
57
58
59

# TODO
# * make it configurable without code
# * better file path attribution
Sylvain Thénault's avatar
Sylvain Thénault committed
60
# * handle backup/restore
61

62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
def uniquify_path(dirpath, basename):
    """return a unique file name for `basename` in `dirpath`, or None
    if all attemps failed.

    XXX subject to race condition.
    """
    path = osp.join(dirpath, basename)
    if not osp.isfile(path):
        return path
    base, ext = osp.splitext(path)
    for i in xrange(1, 256):
        path = '%s%s%s' % (base, i, ext)
        if not osp.isfile(path):
            return path
    return None

78
79
class BytesFileSystemStorage(Storage):
    """store Bytes attribute value on the file system"""
80
    def __init__(self, defaultdir, fsencoding='utf-8'):
81
        self.default_directory = defaultdir
82
        self.fsencoding = fsencoding
83

84
    def callback(self, source, value):
85
86
87
        """sql generator callback when some attribute with a custom storage is
        accessed
        """
88
89
90
91
92
93
        fpath = source.binary_to_str(value)
        try:
            return Binary(file(fpath).read())
        except OSError, ex:
            source.critical("can't open %s: %s", value, ex)
            return None
94
95
96

    def entity_added(self, entity, attr):
        """an entity using this storage for attr has been added"""
97
98
99
100
101
102
103
104
        if entity._cw.transaction_data.get('fs_importing'):
            binary = Binary(file(entity[attr].getvalue()).read())
        else:
            binary = entity.pop(attr)
            fpath = self.new_fs_path(entity, attr)
            # bytes storage used to store file's path
            entity[attr] = Binary(fpath)
            file(fpath, 'w').write(binary.getvalue())
105
            hook.set_operation(entity._cw, 'bfss_added', fpath, AddFileOp)
106
        return binary
107
108
109

    def entity_updated(self, entity, attr):
        """an entity using this storage for attr has been updatded"""
110
111
112
113
        if entity._cw.transaction_data.get('fs_importing'):
            oldpath = self.current_fs_path(entity, attr)
            fpath = entity[attr].getvalue()
            if oldpath != fpath:
114
115
                hook.set_operation(entity._cw, 'bfss_deleted', oldpath,
                                   DeleteFileOp)
116
117
118
119
120
            binary = Binary(file(fpath).read())
        else:
            binary = entity.pop(attr)
            fpath = self.current_fs_path(entity, attr)
            UpdateFileOp(entity._cw, filepath=fpath, filedata=binary.getvalue())
121
        return binary
122
123
124

    def entity_deleted(self, entity, attr):
        """an entity using this storage for attr has been deleted"""
125
126
        fpath = self.current_fs_path(entity, attr)
        hook.set_operation(entity._cw, 'bfss_deleted', fpath, DeleteFileOp)
127
128

    def new_fs_path(self, entity, attr):
129
130
131
132
133
        # We try to get some hint about how to name the file using attribute's
        # name metadata, so we use the real file name and extension when
        # available. Keeping the extension is useful for example in the case of
        # PIL processing that use filename extension to detect content-type, as
        # well as providing more understandable file names on the fs.
134
135
136
        basename = [str(entity.eid), attr]
        name = entity.attr_metadata(attr, 'name')
        if name is not None:
137
            basename.append(name.encode(self.fsencoding))
138
139
140
        fspath = uniquify_path(self.default_directory, '_'.join(basename))
        if fspath is None:
            msg = entity._cw._('failed to uniquify path (%s, %s)') % (
141
                dirpath, '_'.join(basename))
142
            raise ValidationError(entity.eid, {role_name(attr, 'subject'): msg})
143
144
145
        return fspath

    def current_fs_path(self, entity, attr):
146
147
148
        sysource = entity._cw.pool.source('system')
        cu = sysource.doexec(entity._cw,
                             'SELECT cw_%s FROM cw_%s WHERE cw_eid=%s' % (
149
                             attr, entity.__regid__, entity.eid))
150
151
152
153
        rawvalue = cu.fetchone()[0]
        if rawvalue is None: # no previous value
            return self.new_fs_path(entity, attr)
        return sysource._process_value(rawvalue, cu.description[0],
154
                                       binarywrap=str)
155

156
157
158
159
160
161
162
163
164
165
    def migrate_entity(self, entity, attribute):
        """migrate an entity attribute to the storage"""
        entity.edited_attributes = set()
        self.entity_added(entity, attribute)
        session = entity._cw
        source = session.repo.system_source
        attrs = source.preprocess_entity(entity)
        sql = source.sqlgen.update('cw_' + entity.__regid__, attrs,
                                   ['cw_eid'])
        source.doexec(session, sql, attrs)
166

167
168

class AddFileOp(hook.Operation):
169
    def rollback_event(self):
170
171
172
173
174
        for filepath in self.session.transaction_data.pop('bfss_added'):
            try:
                unlink(filepath)
            except Exception, ex:
                self.error('cant remove %s: %s' % (filepath, ex))
175

176
class DeleteFileOp(hook.Operation):
177
    def commit_event(self):
178
179
180
181
182
        for filepath in self.session.transaction_data.pop('bfss_deleted'):
            try:
                unlink(filepath)
            except Exception, ex:
                self.error('cant remove %s: %s' % (filepath, ex))
183

184
class UpdateFileOp(hook.Operation):
185
186
187
    def precommit_event(self):
        try:
            file(self.filepath, 'w').write(self.filedata)
188
189
        except Exception, ex:
            self.exception(str(ex))