devctl.py 32.3 KB
Newer Older
1
# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
21

Adrien Di Mascio's avatar
Adrien Di Mascio committed
22
23
__docformat__ = "restructuredtext en"

24
25
26
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
27
import sys
28
from datetime import datetime
29
from os import mkdir, chdir, path as osp
30
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
31
32

from logilab.common import STD_BLACKLIST
33

Adrien Di Mascio's avatar
Adrien Di Mascio committed
34
from cubicweb.__pkginfo__ import version as cubicwebversion
35
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
36
from cubicweb.cwctl import CWCTL
37
from cubicweb.cwconfig import CubicWebNoAppConfiguration
38
39
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
40
41
42
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

43

44
45
46
47
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
48
    creating = True
49
50
51
52
53
54
    cleanup_interface_sobjects = False

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
55

56
    def __init__(self, *cubes):
57
58
59
60
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
61
            self.load_site_cubicweb()
62
63
        else:
            self._cubes = ()
64

Adrien Di Mascio's avatar
Adrien Di Mascio committed
65
66
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
67
        return None
68
69
70
71

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
72
73
    def main_config_file(self):
        return None
74
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
75
76
77
        pass
    def load_configuration(self):
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
78
79
80
81
82
83
84
85
86
87
88
89
90
    def default_log_file(self):
        return None


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
    for name, mod in sys.modules.items():
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
91
92
93
        if mod.__file__ is None:
            # odd/rare but real
            continue
94
        for path in config.appobjects_path():
Sylvain Thenault's avatar
Sylvain Thenault committed
95
96
97
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
98

Adrien Di Mascio's avatar
Adrien Di Mascio committed
99
100
101
102
103
104
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
105
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
106
    if cubedir:
107
        cube = osp.split(cubedir)[-1]
108
109
110
111
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
112
    else:
113
114
115
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
116
    schema = config.load_schema(remove_unused_rtypes=False)
117
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
118
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
119
120
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
121
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
122
123


124
125
def _generate_schema_pot(w, vreg, schema, libconfig=None):
    from copy import deepcopy
126
    from cubicweb.i18n import add_msg
127
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
128
129
    w('# schema pot file, generated on %s\n'
      % datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
130
131
132
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
133
    vregdone = set()
134
    if libconfig is not None:
135
        from cubicweb.cwvreg import CWRegistryStore
136
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
137
138
        afs = vreg['uicfg'].select('autoform_section')
        appearsin_addmenu = vreg['uicfg'].select('actionbox_appearsin_addmenu')
139
        cleanup_sys_modules(libconfig)
140
        libvreg = CWRegistryStore(libconfig)
141
        libvreg.set_schema(libschema) # trigger objects registration
142
143
        libafs = libvreg['uicfg'].select('autoform_section')
        libappearsin_addmenu = libvreg['uicfg'].select('actionbox_appearsin_addmenu')
144
145
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
146
    else:
147
        libschema = {}
148
149
        afs = vreg['uicfg'].select('autoform_section')
        appearsin_addmenu = vreg['uicfg'].select('actionbox_appearsin_addmenu')
150
151
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
152
    done = set()
153
154
155
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
156
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
157
        etype = eschema.type
158
159
160
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
161
            if not eschema.final:
162
163
164
165
166
                add_msg(w, 'This %s' % etype)
                add_msg(w, 'New %s' % etype)
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
167
        if eschema.final:
168
169
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
170
171
            if rschema.final:
                continue
172
            for tschema in targetschemas:
173
                fsections = afs.etype_get(eschema, rschema, role, tschema)
174
                if 'main_inlined' in fsections and \
175
                       (libconfig is None or not
176
                        'main_inlined' in libafs.etype_get(
177
                            eschema, rschema, role, tschema)):
178
179
                    add_msg(w, 'add a %s' % tschema,
                            'inlined:%s.%s.%s' % (etype, rschema, role))
180
                    add_msg(w, str(tschema),
181
                            'inlined:%s.%s.%s' % (etype, rschema, role))
182
                if appearsin_addmenu.etype_get(eschema, rschema, role, tschema):
183
184
                    if libconfig is not None and libappearsin_addmenu.etype_get(
                        eschema, rschema, role, tschema):
185
186
                        if eschema in libschema and tschema in libschema:
                            continue
187
188
189
190
191
192
193
194
195
196
197
198
                    if role == 'subject':
                        label = 'add %s %s %s %s' % (eschema, rschema,
                                                     tschema, role)
                        label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                            tschema, eschema, rschema, tschema)
                    else:
                        label = 'add %s %s %s %s' % (tschema, rschema,
                                                     eschema, role)
                        label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                            tschema, tschema, rschema, eschema)
                    add_msg(w, label)
                    add_msg(w, label2)
Sylvain Thénault's avatar
Sylvain Thénault committed
199
200
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
201
    w('# subject and object forms for each relation type\n')
202
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
203
    w('\n')
204
205
206
207
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
208
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
209
        rtype = rschema.type
210
        if rtype not in libschema:
211
212
            # bw compat, necessary until all translation of relation are done
            # properly...
213
            add_msg(w, rtype)
214
            done.add(rtype)
215
216
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
217
            done.add(rschema.description)
218
219
220
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
221
        # add context information only for non-metadata rtypes
222
        if rschema not in NO_I18NCONTEXT:
223
            libsubjects = librschema and librschema.subjects() or ()
224
            for subjschema in rschema.subjects():
225
226
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
227
        if not (rschema.final or rschema.symmetric):
228
            if rschema not in NO_I18NCONTEXT:
229
230
231
232
233
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
234
235
                # bw compat, necessary until all translation of relation are
                # done properly...
236
                add_msg(w, '%s_object' % rtype)
237
238
239
240
241
242
243
244
        for rdef in rschema.rdefs.itervalues():
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
245
    for objid in _iter_vreg_objids(vreg, vregdone):
246
247
248
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
249

250
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
251
    for reg, objdict in vreg.items():
252
253
        if reg in ('boxes', 'contentnavigation'):
            continue
254
        for objects in objdict.itervalues():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
255
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
256
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
257
                if objid in done:
258
                    break
259
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
260
                if pdefs:
261
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
262
                    done.add(objid)
263
                    break
264

265

Adrien Di Mascio's avatar
Adrien Di Mascio committed
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
283

Adrien Di Mascio's avatar
Adrien Di Mascio committed
284
285
286
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
287
    name = 'i18ncubicweb'
288
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
289
290
291
292

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
293
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
294
295
        import yams
        from logilab.common.fileutils import ensure_fs_mode
296
        from logilab.common.shellutils import globfind, find, rm
297
        from logilab.common.modutils import get_module_files
298
        from cubicweb.i18n import extract_from_tal, execute2
299
        tempdir = tempfile.mkdtemp(prefix='cw-')
300
        cwi18ndir = WebConfiguration.i18n_lib_dir()
301
302
        print '-> extract messages:',
        print 'schema',
303
304
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
305
306
307
308
309
310
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
        schemapotstream = file(schemapot, 'w')
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
311
        print 'TAL',
312
313
314
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
315
        print '-> generate .pot files.'
316
317
318
319
320
321
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
322
323
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
324
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
325
                                ]:
326
            potfile = osp.join(tempdir, '%s.pot' % id)
327
328
329
330
331
332
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
333
            if osp.exists(potfile):
334
335
                potfiles.append(potfile)
            else:
336
337
                print '-> WARNING: %s file was not generated' % potfile
        print '-> merging %i .pot files' % len(potfiles)
338
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
339
340
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
341
        print '-> merging main pot file with existing translations.'
342
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
343
        toedit = []
344
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
345
            target = '%s.po' % lang
346
347
348
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
349
350
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
351
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
352
        # cleanup
353
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
354
        # instructions pour la suite
355
356
        print '-> regenerated CubicWeb\'s .po catalogs.'
        print '\nYou can now edit the following files:'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
357
        print '* ' + '\n* '.join(toedit)
358
        print 'when you are done, run "cubicweb-ctl i18ncube yourcube".'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
359
360


361
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
362
363
364
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
365
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
366
    arguments = '[<cube>...]'
367

Adrien Di Mascio's avatar
Adrien Di Mascio committed
368
369
370
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
371
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
372
        else:
373
374
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
375
376
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
377
378
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
379

380

Adrien Di Mascio's avatar
Adrien Di Mascio committed
381
def update_cubes_catalogs(cubes):
382
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
383
    for cubedir in cubes:
384
        if not osp.isdir(cubedir):
385
            print '-> ignoring %s that is not a directory.' % cubedir
Adrien Di Mascio's avatar
Adrien Di Mascio committed
386
            continue
387
        try:
388
            toedit = update_cube_catalogs(cubedir)
389
        except CalledProcessError as exc:
390
391
392
            print '\n*** error while updating catalogs for cube', cubedir
            print 'cmd:\n%s' % exc.cmd
            print 'stdout:\n%s\nstderr:\n%s' % exc.data
393
394
395
        except Exception:
            import traceback
            traceback.print_exc()
396
            print '*** error while updating catalogs for cube', cubedir
397
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
398
399
        else:
            # instructions pour la suite
400
401
402
403
404
405
            if toedit:
                print '-> regenerated .po catalogs for cube %s.' % cubedir
                print '\nYou can now edit the following files:'
                print '* ' + '\n* '.join(toedit)
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
406
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
407

408
409
def update_cube_catalogs(cubedir):
    import shutil
410
    import tempfile
411
412
    from logilab.common.fileutils import ensure_fs_mode
    from logilab.common.shellutils import find, rm
413
    from cubicweb.i18n import extract_from_tal, execute2
414
    cube = osp.basename(osp.normpath(cubedir))
415
    tempdir = tempfile.mkdtemp()
416
    print underline_title('Updating i18n catalogs for cube %s' % cube)
417
    chdir(cubedir)
418
    if osp.exists(osp.join('i18n', 'entities.pot')):
419
        warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'
420
421
422
423
             % osp.join('i18n', 'entities.pot'), DeprecationWarning)
        potfiles = [osp.join('i18n', 'entities.pot')]
    elif osp.exists(osp.join('i18n', 'static-messages.pot')):
        potfiles = [osp.join('i18n', 'static-messages.pot')]
424
425
    else:
        potfiles = []
426
427
    print '-> extracting messages:',
    print 'schema',
428
    schemapot = osp.join(tempdir, 'schema.pot')
429
430
431
432
433
434
    potfiles.append(schemapot)
    # explicit close necessary else the file may not be yet flushed when
    # we'll using it below
    schemapotstream = file(schemapot, 'w')
    generate_schema_pot(schemapotstream.write, cubedir)
    schemapotstream.close()
435
    print 'TAL',
436
437
438
    tali18nfile = osp.join(tempdir, 'tali18n.py')
    ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST+('test',))
    extract_from_tal(ptfiles, tali18nfile)
439
    print 'Javascript'
440
441
    jsfiles =  [jsfile for jsfile in find('.', '.js')
                if osp.basename(jsfile).startswith('cub')]
442
    if jsfiles:
443
        tmppotfile = osp.join(tempdir, 'js.pot')
444
445
446
        cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-L', 'java',
               '--from-code=utf-8', '-o', tmppotfile] + jsfiles
        execute2(cmd)
447
        # no pot file created if there are no string to translate
448
        if osp.exists(tmppotfile):
449
            potfiles.append(tmppotfile)
450
    print '-> creating cube-specific catalog'
451
    tmppotfile = osp.join(tempdir, 'generated.pot')
452
453
    cubefiles = find('.', '.py', blacklist=STD_BLACKLIST+('test',))
    cubefiles.append(tali18nfile)
454
455
456
    cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-o', tmppotfile]
    cmd.extend(cubefiles)
    execute2(cmd)
457
    if osp.exists(tmppotfile): # doesn't exists of no translation string found
458
        potfiles.append(tmppotfile)
459
    potfile = osp.join(tempdir, 'cube.pot')
460
461
462
463
    print '-> merging %i .pot files' % len(potfiles)
    cmd = ['msgcat', '-o', potfile]
    cmd.extend(potfiles)
    execute2(cmd)
464
    if not osp.exists(potfile):
465
466
467
468
        print 'no message catalog for cube', cube, 'nothing to translate'
        # cleanup
        rm(tempdir)
        return ()
469
    print '-> merging main pot file with existing translations:',
470
    chdir('i18n')
471
    toedit = []
472
    for lang in CubicWebNoAppConfiguration.cw_languages():
473
        print lang,
474
        cubepo = '%s.po' % lang
475
        if not osp.exists(cubepo):
476
477
            shutil.copy(potfile, cubepo)
        else:
478
479
            cmd = ['msgmerge','-N','-s','-o', cubepo+'new', cubepo, potfile]
            execute2(cmd)
480
481
            ensure_fs_mode(cubepo)
            shutil.move('%snew' % cubepo, cubepo)
482
        toedit.append(osp.abspath(cubepo))
483
    print
484
485
486
487
488
    # cleanup
    rm(tempdir)
    return toedit


489
490
491
492
493
494
495
# XXX totally broken, fix it
# class LiveServerCommand(Command):
#     """Run a server from within a cube directory.
#     """
#     name = 'live-server'
#     arguments = ''
#     options = ()
496

497
498
499
500
#     def run(self, args):
#         """run the command with its specific arguments"""
#         from cubicweb.devtools.livetest import runserver
#         runserver()
Adrien Di Mascio's avatar
Adrien Di Mascio committed
501
502


503
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
504
505
506
    """Create a new cube.

    <cubename>
507
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
508
509
510
    """
    name = 'newcube'
    arguments = '<cubename>'
511
    min_args = max_args = 1
512
    options = (
513
514
515
516
517
518
519
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
520
521
522
523
524
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
525
526
527
528
529
530
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
549
550
551
552
553
554
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
555
556
        )

557
    LICENSES = {
558
559
560
561
562
563
564
565
566
567
568
569
570
571
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
''',
572

573
574
575
576
577
578
579
580
581
582
583
584
585
586
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
587
        }
588

Adrien Di Mascio's avatar
Adrien Di Mascio committed
589
    def run(self, args):
590
        import re
591
        from logilab.common.shellutils import ASK
592
593
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
594
595
            raise BadCommandUsage(
                'cube name must be a valid python module name')
596
        verbose = self.get('verbose')
597
        cubesdir = self.get('directory')
598
599
        if not cubesdir:
            cubespath = ServerConfiguration.cubes_search_path()
600
            if len(cubespath) > 1:
601
602
603
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
604
            cubesdir = cubespath[0]
605
        if not osp.isdir(cubesdir):
606
            print "-> creating cubes directory", cubesdir
Adrien Di Mascio's avatar
Adrien Di Mascio committed
607
            try:
608
                mkdir(cubesdir)
609
            except OSError as err:
610
611
612
613
614
615
                self.fail("failed to create directory %r\n(%s)"
                          % (cubesdir, err))
        cubedir = osp.join(cubesdir, cubename)
        if osp.exists(cubedir):
            self.fail("%s already exists !" % cubedir)
        skeldir = osp.join(BASEDIR, 'skeleton')
616
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
617
        if verbose:
618
619
            distname = raw_input('Debian name for your cube ? [%s]): '
                                 % default_name).strip()
620
            if not distname:
621
                distname = default_name
622
            elif not distname.startswith('cubicweb-'):
623
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
624
625
                    distname = 'cubicweb-' + distname
        else:
626
            distname = default_name
627
        if not re.match('[a-z][-a-z0-9]*$', distname):
628
629
630
631
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
        longdesc = shortdesc = raw_input(
            'Enter a short description for your cube: ')
632
        if verbose:
633
634
            longdesc = raw_input(
                'Enter a long description (leave empty to reuse the short one): ')
635
        dependencies = {'cubicweb': '>= %s' % cubicwebversion}
636
        if verbose:
637
            dependencies.update(self._ask_for_dependencies())
Adrien Di Mascio's avatar
Adrien Di Mascio committed
638
639
640
641
        context = {'cubename' : cubename,
                   'distname' : distname,
                   'shortdesc' : shortdesc,
                   'longdesc' : longdesc or shortdesc,
642
                   'dependencies' : dependencies,
Adrien Di Mascio's avatar
Adrien Di Mascio committed
643
                   'version'  : cubicwebversion,
644
                   'year'  : str(datetime.now().year),
645
646
647
                   'author': self['author'],
                   'author-email': self['author-email'],
                   'author-web-site': self['author-web-site'],
648
649
                   'license': self['license'],
                   'long-license': self.LICENSES[self['license']],
Adrien Di Mascio's avatar
Adrien Di Mascio committed
650
                   }
651
652
653
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
654
                        'cubes.*', 'uiprops.py*')
655
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
656

657
    def _ask_for_dependencies(self):
658
659
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
660
661
662
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
663
664
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
665
                depcubes.append(cube)
666
            if answer == 'type':
667
                depcubes = splitstrip(raw_input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
668
                break
669
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
670
                break
671
672
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
673

674
675
676
677

class ExamineLogCommand(Command):
    """Examine a rql log file.

678
    Will print out the following table
679

680
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
681

682
    sorted by descending cumulative time (clock). Time are expressed in seconds.
683

684
    Chances are the lines at the top are the ones that will bring the higher
685
    benefit after optimisation. Start there.
686
    """
687
    arguments = 'rql.log'
688
    name = 'exlog'
689
    options = ()
690

691
692
693
    def run(self, args):
        import re
        requests = {}
694
        for filepath in args:
695
            try:
696
                stream = file(filepath)
697
            except OSError as ex:
698
699
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
700
            for lineno, line in enumerate(stream):
701
702
703
704
705
706
707
708
709
710
711
712
713
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
714
                except Exception as exc:
715
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
716
        stat = []
717
        for rql, times in requests.iteritems():
718
719
720
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
721
722
        stat.sort()
        stat.reverse()
723
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
724
725
        print 'Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query'
        for clocktime, cputime, occ, rql in stat:
726
727
            print '%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql)
728

729

730
731
732
733
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
734
    min_args = max_args = 1
735
736
    options = [
        ('output-file',
737
         {'type':'string', 'default': None,
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
        ]
772
773

    def run(self, args):
774
775
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
776
        from logilab.common.textutils import splitstrip
777
778
779
        from yams import schema2dot, BASE_TYPES
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
780
        cubes = splitstrip(args[0])
781
        dev_conf = DevConfiguration(*cubes)
782
783
784
785
786
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
787
788
789
790
791
792
793
794
795
796
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
        schema2dot.schema2dot(schema, out, skiptypes=skiptypes)
797
798
799
800
        if viewer:
            p = Popen((viewer, out))
            p.wait()

801
802
803
804
805
806
807
808
809
810

class GenerateQUnitHTML(Command):
    """Generate a QUnit html file to see test in your browser"""
    name = "qunit-html"
    arguments = '<test file> [<dependancy js file>...]'

    def run(self, args):
        from cubicweb.devtools.qunit import make_qunit_html
        print make_qunit_html(args[0], args[1:])

811
for cmdcls in (UpdateCubicWebCatalogCommand,
812
               UpdateCubeCatalogCommand,
813
814
815
816
817
818
819
               #LiveServerCommand,
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               GenerateQUnitHTML,
               ):
    CWCTL.register(cmdcls)