devctl.py 33.5 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
21

Adrien Di Mascio's avatar
Adrien Di Mascio committed
22
23
__docformat__ = "restructuredtext en"

24
25
26
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
27
import sys
28
from datetime import datetime
29
from os import mkdir, chdir, path as osp
30
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
31
32

from logilab.common import STD_BLACKLIST
33

Adrien Di Mascio's avatar
Adrien Di Mascio committed
34
from cubicweb.__pkginfo__ import version as cubicwebversion
35
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
36
from cubicweb.cwctl import CWCTL
37
from cubicweb.cwconfig import CubicWebNoAppConfiguration
38
39
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
40
41
42
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

43

44
45
46
47
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
48
    creating = True
49
    cleanup_unused_appobjects = False
50
51
52
53
54

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
55

56
    def __init__(self, *cubes):
57
58
59
60
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
61
            self.load_site_cubicweb()
62
63
        else:
            self._cubes = ()
64

Adrien Di Mascio's avatar
Adrien Di Mascio committed
65
66
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
67
        return None
68
69
70
71

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
72
73
    def main_config_file(self):
        return None
74
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
75
        pass
76
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
77
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
78
79
    def default_log_file(self):
        return None
80
81
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
82
83
84
85
86
87
88
89
90
91
92


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
    for name, mod in sys.modules.items():
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
93
94
95
        if mod.__file__ is None:
            # odd/rare but real
            continue
96
        for path in config.appobjects_path():
Sylvain Thenault's avatar
Sylvain Thenault committed
97
98
99
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
100

Adrien Di Mascio's avatar
Adrien Di Mascio committed
101
102
103
104
105
106
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
107
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
108
    if cubedir:
109
        cube = osp.split(cubedir)[-1]
110
111
112
113
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
114
    else:
115
116
117
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
118
    schema = config.load_schema(remove_unused_rtypes=False)
119
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
120
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
121
122
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
123
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
124
125


126
def _generate_schema_pot(w, vreg, schema, libconfig=None):
127
    from cubicweb.i18n import add_msg
128
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
129
130
    w('# schema pot file, generated on %s\n'
      % datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
131
132
133
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
134
    vregdone = set()
135
    afss = vreg['uicfg']['autoform_section']
136
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
137
    if libconfig is not None:
138
139
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
140
        from cubicweb.cwvreg import CWRegistryStore
141
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
142
        cleanup_sys_modules(libconfig)
143
        libvreg = CWRegistryStore(libconfig)
144
        libvreg.set_schema(libschema) # trigger objects registration
145
        libafss = libvreg['uicfg']['autoform_section']
146
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
147
148
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
149
150
151
152

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
153
    else:
154
        # processing cubicweb itself
155
        libschema = {}
156
157
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
158
159
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
160
    done = set()
161
162
163
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
164
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
165
        etype = eschema.type
166
167
168
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
169
            if not eschema.final:
170
                add_msg(w, 'This %s:' % etype)
171
                add_msg(w, 'New %s' % etype)
172
                add_msg(w, 'add a %s' % etype) # AddNewAction
173
174
175
176
177
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
178
179
180
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
181
        if eschema.final:
182
183
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
184
185
            if rschema.final:
                continue
186
            for tschema in targetschemas:
187

188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
215
216
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
217
    w('# subject and object forms for each relation type\n')
218
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
219
    w('\n')
220
221
222
223
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
224
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
225
        rtype = rschema.type
226
        if rtype not in libschema:
227
228
            # bw compat, necessary until all translation of relation are done
            # properly...
229
            add_msg(w, rtype)
230
            done.add(rtype)
231
232
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
233
            done.add(rschema.description)
234
235
236
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
237
        # add context information only for non-metadata rtypes
238
        if rschema not in NO_I18NCONTEXT:
239
            libsubjects = librschema and librschema.subjects() or ()
240
            for subjschema in rschema.subjects():
241
242
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
243
        if not (rschema.final or rschema.symmetric):
244
            if rschema not in NO_I18NCONTEXT:
245
246
247
248
249
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
250
251
                # bw compat, necessary until all translation of relation are
                # done properly...
252
                add_msg(w, '%s_object' % rtype)
253
254
255
256
257
258
259
260
        for rdef in rschema.rdefs.itervalues():
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
261
    for objid in _iter_vreg_objids(vreg, vregdone):
262
263
264
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
265

266
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
267
    for reg, objdict in vreg.items():
268
269
        if reg in ('boxes', 'contentnavigation'):
            continue
270
        for objects in objdict.itervalues():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
271
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
272
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
273
                if objid in done:
274
                    break
275
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
276
                if pdefs:
277
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
278
                    done.add(objid)
279
                    break
280

281

Adrien Di Mascio's avatar
Adrien Di Mascio committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
299

Adrien Di Mascio's avatar
Adrien Di Mascio committed
300
301
302
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
303
    name = 'i18ncubicweb'
304
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
305
306
307
308

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
309
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
310
311
        import yams
        from logilab.common.fileutils import ensure_fs_mode
312
        from logilab.common.shellutils import globfind, find, rm
313
        from logilab.common.modutils import get_module_files
314
        from cubicweb.i18n import extract_from_tal, execute2
315
        tempdir = tempfile.mkdtemp(prefix='cw-')
316
        cwi18ndir = WebConfiguration.i18n_lib_dir()
317
318
        print '-> extract messages:',
        print 'schema',
319
320
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
321
322
323
324
325
326
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
        schemapotstream = file(schemapot, 'w')
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
327
        print 'TAL',
328
329
330
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
331
        print '-> generate .pot files.'
332
333
334
335
336
337
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
338
339
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
340
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
341
                                ]:
342
            potfile = osp.join(tempdir, '%s.pot' % id)
343
344
345
346
347
348
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
349
            if osp.exists(potfile):
350
351
                potfiles.append(potfile)
            else:
352
353
                print '-> WARNING: %s file was not generated' % potfile
        print '-> merging %i .pot files' % len(potfiles)
354
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
355
356
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
357
        print '-> merging main pot file with existing translations.'
358
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
359
        toedit = []
360
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
361
            target = '%s.po' % lang
362
363
364
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
365
366
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
367
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
368
        # cleanup
369
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
370
        # instructions pour la suite
371
372
        print '-> regenerated CubicWeb\'s .po catalogs.'
        print '\nYou can now edit the following files:'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
373
        print '* ' + '\n* '.join(toedit)
374
        print 'when you are done, run "cubicweb-ctl i18ncube yourcube".'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
375
376


377
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
378
379
380
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
381
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
382
    arguments = '[<cube>...]'
383

Adrien Di Mascio's avatar
Adrien Di Mascio committed
384
385
386
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
387
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
388
        else:
389
390
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
391
392
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
393
394
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
395

396

Adrien Di Mascio's avatar
Adrien Di Mascio committed
397
def update_cubes_catalogs(cubes):
398
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
399
    for cubedir in cubes:
400
        if not osp.isdir(cubedir):
401
            print '-> ignoring %s that is not a directory.' % cubedir
Adrien Di Mascio's avatar
Adrien Di Mascio committed
402
            continue
403
        try:
404
            toedit = update_cube_catalogs(cubedir)
405
        except CalledProcessError as exc:
406
407
408
            print '\n*** error while updating catalogs for cube', cubedir
            print 'cmd:\n%s' % exc.cmd
            print 'stdout:\n%s\nstderr:\n%s' % exc.data
409
410
411
        except Exception:
            import traceback
            traceback.print_exc()
412
            print '*** error while updating catalogs for cube', cubedir
413
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
414
415
        else:
            # instructions pour la suite
416
417
418
419
420
421
            if toedit:
                print '-> regenerated .po catalogs for cube %s.' % cubedir
                print '\nYou can now edit the following files:'
                print '* ' + '\n* '.join(toedit)
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
422
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
423

424
425
def update_cube_catalogs(cubedir):
    import shutil
426
    import tempfile
427
428
    from logilab.common.fileutils import ensure_fs_mode
    from logilab.common.shellutils import find, rm
429
    from cubicweb.i18n import extract_from_tal, execute2
430
    cube = osp.basename(osp.normpath(cubedir))
431
    tempdir = tempfile.mkdtemp()
432
    print underline_title('Updating i18n catalogs for cube %s' % cube)
433
    chdir(cubedir)
434
    if osp.exists(osp.join('i18n', 'entities.pot')):
435
        warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'
436
437
438
439
             % osp.join('i18n', 'entities.pot'), DeprecationWarning)
        potfiles = [osp.join('i18n', 'entities.pot')]
    elif osp.exists(osp.join('i18n', 'static-messages.pot')):
        potfiles = [osp.join('i18n', 'static-messages.pot')]
440
441
    else:
        potfiles = []
442
443
    print '-> extracting messages:',
    print 'schema',
444
    schemapot = osp.join(tempdir, 'schema.pot')
445
446
447
448
449
450
    potfiles.append(schemapot)
    # explicit close necessary else the file may not be yet flushed when
    # we'll using it below
    schemapotstream = file(schemapot, 'w')
    generate_schema_pot(schemapotstream.write, cubedir)
    schemapotstream.close()
451
    print 'TAL',
452
453
454
    tali18nfile = osp.join(tempdir, 'tali18n.py')
    ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST+('test',))
    extract_from_tal(ptfiles, tali18nfile)
455
    print 'Javascript'
456
457
    jsfiles =  [jsfile for jsfile in find('.', '.js')
                if osp.basename(jsfile).startswith('cub')]
458
    if jsfiles:
459
        tmppotfile = osp.join(tempdir, 'js.pot')
460
461
462
        cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-L', 'java',
               '--from-code=utf-8', '-o', tmppotfile] + jsfiles
        execute2(cmd)
463
        # no pot file created if there are no string to translate
464
        if osp.exists(tmppotfile):
465
            potfiles.append(tmppotfile)
466
    print '-> creating cube-specific catalog'
467
    tmppotfile = osp.join(tempdir, 'generated.pot')
468
469
    cubefiles = find('.', '.py', blacklist=STD_BLACKLIST+('test',))
    cubefiles.append(tali18nfile)
470
471
472
    cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-o', tmppotfile]
    cmd.extend(cubefiles)
    execute2(cmd)
473
    if osp.exists(tmppotfile): # doesn't exists of no translation string found
474
        potfiles.append(tmppotfile)
475
    potfile = osp.join(tempdir, 'cube.pot')
476
477
478
479
    print '-> merging %i .pot files' % len(potfiles)
    cmd = ['msgcat', '-o', potfile]
    cmd.extend(potfiles)
    execute2(cmd)
480
    if not osp.exists(potfile):
481
482
483
484
        print 'no message catalog for cube', cube, 'nothing to translate'
        # cleanup
        rm(tempdir)
        return ()
485
    print '-> merging main pot file with existing translations:',
486
    chdir('i18n')
487
    toedit = []
488
    for lang in CubicWebNoAppConfiguration.cw_languages():
489
        print lang,
490
        cubepo = '%s.po' % lang
491
        if not osp.exists(cubepo):
492
493
            shutil.copy(potfile, cubepo)
        else:
494
495
            cmd = ['msgmerge','-N','-s','-o', cubepo+'new', cubepo, potfile]
            execute2(cmd)
496
497
            ensure_fs_mode(cubepo)
            shutil.move('%snew' % cubepo, cubepo)
498
        toedit.append(osp.abspath(cubepo))
499
    print
500
501
502
503
504
    # cleanup
    rm(tempdir)
    return toedit


505
506
507
508
509
510
511
# XXX totally broken, fix it
# class LiveServerCommand(Command):
#     """Run a server from within a cube directory.
#     """
#     name = 'live-server'
#     arguments = ''
#     options = ()
512

513
514
515
516
#     def run(self, args):
#         """run the command with its specific arguments"""
#         from cubicweb.devtools.livetest import runserver
#         runserver()
Adrien Di Mascio's avatar
Adrien Di Mascio committed
517
518


519
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
520
521
522
    """Create a new cube.

    <cubename>
523
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
524
525
526
    """
    name = 'newcube'
    arguments = '<cubename>'
527
    min_args = max_args = 1
528
    options = (
529
530
531
532
533
534
535
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
536
537
538
539
540
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
541
542
543
544
545
546
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
565
566
567
568
569
570
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
571
572
        )

573
    LICENSES = {
574
575
576
577
578
579
580
581
582
583
584
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
585
586
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
587
''',
588

589
590
591
592
593
594
595
596
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
597
598
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
599
600
601
602
603
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
604
        }
605

Adrien Di Mascio's avatar
Adrien Di Mascio committed
606
    def run(self, args):
607
        import re
608
        from logilab.common.shellutils import ASK
609
610
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
611
612
            raise BadCommandUsage(
                'cube name must be a valid python module name')
613
        verbose = self.get('verbose')
614
        cubesdir = self.get('directory')
615
616
        if not cubesdir:
            cubespath = ServerConfiguration.cubes_search_path()
617
            if len(cubespath) > 1:
618
619
620
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
621
            cubesdir = cubespath[0]
622
        if not osp.isdir(cubesdir):
623
            print "-> creating cubes directory", cubesdir
Adrien Di Mascio's avatar
Adrien Di Mascio committed
624
            try:
625
                mkdir(cubesdir)
626
            except OSError as err:
627
628
629
630
                self.fail("failed to create directory %r\n(%s)"
                          % (cubesdir, err))
        cubedir = osp.join(cubesdir, cubename)
        if osp.exists(cubedir):
Dimitri Papadopoulos's avatar
Dimitri Papadopoulos committed
631
            self.fail("%s already exists!" % cubedir)
632
        skeldir = osp.join(BASEDIR, 'skeleton')
633
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
634
        if verbose:
635
636
            distname = raw_input('Debian name for your cube ? [%s]): '
                                 % default_name).strip()
637
            if not distname:
638
                distname = default_name
639
            elif not distname.startswith('cubicweb-'):
640
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
641
642
                    distname = 'cubicweb-' + distname
        else:
643
            distname = default_name
644
        if not re.match('[a-z][-a-z0-9]*$', distname):
645
646
647
648
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
        longdesc = shortdesc = raw_input(
            'Enter a short description for your cube: ')
649
        if verbose:
650
651
            longdesc = raw_input(
                'Enter a long description (leave empty to reuse the short one): ')
652
        dependencies = {'cubicweb': '>= %s' % cubicwebversion}
653
        if verbose:
654
            dependencies.update(self._ask_for_dependencies())
Adrien Di Mascio's avatar
Adrien Di Mascio committed
655
656
657
658
        context = {'cubename' : cubename,
                   'distname' : distname,
                   'shortdesc' : shortdesc,
                   'longdesc' : longdesc or shortdesc,
659
                   'dependencies' : dependencies,
Adrien Di Mascio's avatar
Adrien Di Mascio committed
660
                   'version'  : cubicwebversion,
661
                   'year'  : str(datetime.now().year),
662
663
664
                   'author': self['author'],
                   'author-email': self['author-email'],
                   'author-web-site': self['author-web-site'],
665
666
                   'license': self['license'],
                   'long-license': self.LICENSES[self['license']],
Adrien Di Mascio's avatar
Adrien Di Mascio committed
667
                   }
668
669
670
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
671
                        'cubes.*', 'uiprops.py*')
672
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
673

674
    def _ask_for_dependencies(self):
675
676
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
677
678
679
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
680
681
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
682
                depcubes.append(cube)
683
            if answer == 'type':
684
                depcubes = splitstrip(raw_input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
685
                break
686
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
687
                break
688
689
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
690

691
692
693
694

class ExamineLogCommand(Command):
    """Examine a rql log file.

695
    Will print out the following table
696

697
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
698

699
    sorted by descending cumulative time (clock). Time are expressed in seconds.
700

701
    Chances are the lines at the top are the ones that will bring the higher
702
    benefit after optimisation. Start there.
703
    """
704
    arguments = 'rql.log'
705
    name = 'exlog'
706
    options = ()
707

708
709
710
    def run(self, args):
        import re
        requests = {}
711
        for filepath in args:
712
            try:
713
                stream = file(filepath)
714
            except OSError as ex:
715
716
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
717
            for lineno, line in enumerate(stream):
718
719
720
721
722
723
724
725
726
727
728
729
730
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
731
                except Exception as exc:
732
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
733
        stat = []
734
        for rql, times in requests.iteritems():
735
736
737
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
738
739
        stat.sort()
        stat.reverse()
740
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
741
742
        print 'Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query'
        for clocktime, cputime, occ, rql in stat:
743
744
            print '%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql)
745

746

747
748
749
750
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
751
    min_args = max_args = 1
752
753
    options = [
        ('output-file',
754
         {'type':'string', 'default': None,
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
788
789
790
791
792
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
793
        ]
794
795

    def run(self, args):
796
797
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
798
        from logilab.common.textutils import splitstrip
799
800
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
801
802
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
803
        cubes = splitstrip(args[0])
804
        dev_conf = DevConfiguration(*cubes)
805
806
807
808
809
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
810
811
812
813
814
815
816
817
818
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

835
836
837
838
        if viewer:
            p = Popen((viewer, out))
            p.wait()

839

840
for cmdcls in (UpdateCubicWebCatalogCommand,
841
               UpdateCubeCatalogCommand,
842
843
844
845
846
847
               #LiveServerCommand,
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)