devctl.py 33.7 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

Adrien Di Mascio's avatar
Adrien Di Mascio committed
23
24
__docformat__ = "restructuredtext en"

25
26
27
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
28
import sys
29
from datetime import datetime, date
30
from os import mkdir, chdir, path as osp
31
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
32
33

from logilab.common import STD_BLACKLIST
34

Adrien Di Mascio's avatar
Adrien Di Mascio committed
35
from cubicweb.__pkginfo__ import version as cubicwebversion
36
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
37
from cubicweb.cwctl import CWCTL
38
from cubicweb.cwconfig import CubicWebNoAppConfiguration
39
40
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
41
42
43
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

44

45
46
47
48
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
49
    creating = True
50
    cleanup_unused_appobjects = False
51
52
53
54
55

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
56

57
    def __init__(self, *cubes):
58
59
60
61
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
62
            self.load_site_cubicweb()
63
64
        else:
            self._cubes = ()
65

Adrien Di Mascio's avatar
Adrien Di Mascio committed
66
67
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
68
        return None
69
70
71
72

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
73
74
    def main_config_file(self):
        return None
75
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
76
        pass
77
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
78
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
79
80
    def default_log_file(self):
        return None
81
82
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
83
84
85
86


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
87
    for name, mod in list(sys.modules.items()):
Sylvain Thenault's avatar
Sylvain Thenault committed
88
89
90
91
92
93
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
94
95
96
        if mod.__file__ is None:
            # odd/rare but real
            continue
97
        for path in config.appobjects_path():
Sylvain Thenault's avatar
Sylvain Thenault committed
98
99
100
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
101

Adrien Di Mascio's avatar
Adrien Di Mascio committed
102
103
104
105
106
107
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
108
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
109
    if cubedir:
110
        cube = osp.split(cubedir)[-1]
111
112
113
114
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
115
    else:
116
117
118
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
119
    schema = config.load_schema(remove_unused_rtypes=False)
120
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
121
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
122
123
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
124
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
125
126


127
def _generate_schema_pot(w, vreg, schema, libconfig=None):
128
    from cubicweb.i18n import add_msg
129
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
130
    w('# schema pot file, generated on %s\n'
131
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
132
133
134
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
135
    vregdone = set()
136
    afss = vreg['uicfg']['autoform_section']
137
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
138
    if libconfig is not None:
139
140
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
141
        from cubicweb.cwvreg import CWRegistryStore
142
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
143
        cleanup_sys_modules(libconfig)
144
        libvreg = CWRegistryStore(libconfig)
145
        libvreg.set_schema(libschema) # trigger objects registration
146
        libafss = libvreg['uicfg']['autoform_section']
147
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
148
149
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
150
151
152
153

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
154
    else:
155
        # processing cubicweb itself
156
        libschema = {}
157
158
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
159
160
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
161
    done = set()
162
163
164
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
165
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
166
        etype = eschema.type
167
168
169
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
170
            if not eschema.final:
171
                add_msg(w, 'This %s:' % etype)
172
                add_msg(w, 'New %s' % etype)
173
                add_msg(w, 'add a %s' % etype) # AddNewAction
174
175
176
177
178
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
179
180
181
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
182
        if eschema.final:
183
184
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
185
186
            if rschema.final:
                continue
187
            for tschema in targetschemas:
188

189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
216
217
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
218
    w('# subject and object forms for each relation type\n')
219
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
220
    w('\n')
221
222
223
224
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
225
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
226
        rtype = rschema.type
227
        if rtype not in libschema:
228
229
            # bw compat, necessary until all translation of relation are done
            # properly...
230
            add_msg(w, rtype)
231
            done.add(rtype)
232
233
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
234
            done.add(rschema.description)
235
236
237
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
238
        # add context information only for non-metadata rtypes
239
        if rschema not in NO_I18NCONTEXT:
240
            libsubjects = librschema and librschema.subjects() or ()
241
            for subjschema in rschema.subjects():
242
243
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
244
        if not (rschema.final or rschema.symmetric):
245
            if rschema not in NO_I18NCONTEXT:
246
247
248
249
250
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
251
252
                # bw compat, necessary until all translation of relation are
                # done properly...
253
                add_msg(w, '%s_object' % rtype)
254
        for rdef in rschema.rdefs.values():
255
256
257
258
259
260
261
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
262
    for objid in _iter_vreg_objids(vreg, vregdone):
263
264
265
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
266

267
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
268
    for reg, objdict in vreg.items():
269
270
        if reg in ('boxes', 'contentnavigation'):
            continue
271
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
272
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
273
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
274
                if objid in done:
275
                    break
276
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
277
                if pdefs:
278
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
279
                    done.add(objid)
280
                    break
281

282

Adrien Di Mascio's avatar
Adrien Di Mascio committed
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
300

Adrien Di Mascio's avatar
Adrien Di Mascio committed
301
302
303
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
304
    name = 'i18ncubicweb'
305
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
306
307
308
309

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
310
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
311
312
        import yams
        from logilab.common.fileutils import ensure_fs_mode
313
        from logilab.common.shellutils import globfind, find, rm
314
        from logilab.common.modutils import get_module_files
315
        from cubicweb.i18n import extract_from_tal, execute2
316
        tempdir = tempfile.mkdtemp(prefix='cw-')
317
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
318
319
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
320
321
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
322
323
324
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
325
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
326
327
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
328
        print('TAL', end=' ')
329
330
331
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
332
        print('-> generate .pot files.')
333
334
335
336
337
338
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
339
340
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
341
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
342
                                ]:
343
            potfile = osp.join(tempdir, '%s.pot' % id)
344
345
346
347
348
349
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
350
            if osp.exists(potfile):
351
352
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
353
354
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
355
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
356
357
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
358
        print('-> merging main pot file with existing translations.')
359
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
360
        toedit = []
361
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
362
            target = '%s.po' % lang
363
364
365
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
366
367
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
368
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
369
        # cleanup
370
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
371
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
372
373
374
375
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
376
377


378
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
379
380
381
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
382
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
383
    arguments = '[<cube>...]'
384

Adrien Di Mascio's avatar
Adrien Di Mascio committed
385
386
387
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
388
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
389
        else:
390
391
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
392
393
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
394
395
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
396

397

Adrien Di Mascio's avatar
Adrien Di Mascio committed
398
def update_cubes_catalogs(cubes):
399
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
400
    for cubedir in cubes:
401
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
402
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
403
            continue
404
        try:
405
            toedit = update_cube_catalogs(cubedir)
406
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
407
408
409
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
410
411
412
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
413
            print('*** error while updating catalogs for cube', cubedir)
414
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
415
416
        else:
            # instructions pour la suite
417
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
418
419
420
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
421
422
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
423
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
424

425
426
def update_cube_catalogs(cubedir):
    import shutil
427
    import tempfile
428
429
    from logilab.common.fileutils import ensure_fs_mode
    from logilab.common.shellutils import find, rm
430
    from cubicweb.i18n import extract_from_tal, execute2
431
    cube = osp.basename(osp.normpath(cubedir))
432
    tempdir = tempfile.mkdtemp()
Samuel Trégouët's avatar
Samuel Trégouët committed
433
    print(underline_title('Updating i18n catalogs for cube %s' % cube))
434
    chdir(cubedir)
435
    if osp.exists(osp.join('i18n', 'entities.pot')):
436
        warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'
437
438
439
440
             % osp.join('i18n', 'entities.pot'), DeprecationWarning)
        potfiles = [osp.join('i18n', 'entities.pot')]
    elif osp.exists(osp.join('i18n', 'static-messages.pot')):
        potfiles = [osp.join('i18n', 'static-messages.pot')]
441
442
    else:
        potfiles = []
Samuel Trégouët's avatar
Samuel Trégouët committed
443
444
    print('-> extracting messages:', end=' ')
    print('schema', end=' ')
445
    schemapot = osp.join(tempdir, 'schema.pot')
446
447
448
    potfiles.append(schemapot)
    # explicit close necessary else the file may not be yet flushed when
    # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
449
    schemapotstream = open(schemapot, 'w')
450
451
    generate_schema_pot(schemapotstream.write, cubedir)
    schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
452
    print('TAL', end=' ')
453
454
455
    tali18nfile = osp.join(tempdir, 'tali18n.py')
    ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST+('test',))
    extract_from_tal(ptfiles, tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
456
    print('Javascript')
457
458
    jsfiles =  [jsfile for jsfile in find('.', '.js')
                if osp.basename(jsfile).startswith('cub')]
459
    if jsfiles:
460
        tmppotfile = osp.join(tempdir, 'js.pot')
461
462
463
        cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-L', 'java',
               '--from-code=utf-8', '-o', tmppotfile] + jsfiles
        execute2(cmd)
464
        # no pot file created if there are no string to translate
465
        if osp.exists(tmppotfile):
466
            potfiles.append(tmppotfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
467
    print('-> creating cube-specific catalog')
468
    tmppotfile = osp.join(tempdir, 'generated.pot')
469
470
    cubefiles = find('.', '.py', blacklist=STD_BLACKLIST+('test',))
    cubefiles.append(tali18nfile)
471
472
473
    cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-o', tmppotfile]
    cmd.extend(cubefiles)
    execute2(cmd)
474
    if osp.exists(tmppotfile): # doesn't exists of no translation string found
475
        potfiles.append(tmppotfile)
476
    potfile = osp.join(tempdir, 'cube.pot')
Samuel Trégouët's avatar
Samuel Trégouët committed
477
    print('-> merging %i .pot files' % len(potfiles))
478
479
480
    cmd = ['msgcat', '-o', potfile]
    cmd.extend(potfiles)
    execute2(cmd)
481
    if not osp.exists(potfile):
Samuel Trégouët's avatar
Samuel Trégouët committed
482
        print('no message catalog for cube', cube, 'nothing to translate')
483
484
485
        # cleanup
        rm(tempdir)
        return ()
Samuel Trégouët's avatar
Samuel Trégouët committed
486
    print('-> merging main pot file with existing translations:', end=' ')
487
    chdir('i18n')
488
    toedit = []
489
    for lang in CubicWebNoAppConfiguration.cw_languages():
Samuel Trégouët's avatar
Samuel Trégouët committed
490
        print(lang, end=' ')
491
        cubepo = '%s.po' % lang
492
        if not osp.exists(cubepo):
493
494
            shutil.copy(potfile, cubepo)
        else:
495
496
            cmd = ['msgmerge','-N','-s','-o', cubepo+'new', cubepo, potfile]
            execute2(cmd)
497
498
            ensure_fs_mode(cubepo)
            shutil.move('%snew' % cubepo, cubepo)
499
        toedit.append(osp.abspath(cubepo))
Samuel Trégouët's avatar
Samuel Trégouët committed
500
    print()
501
502
503
504
505
    # cleanup
    rm(tempdir)
    return toedit


506
507
508
509
510
511
512
# XXX totally broken, fix it
# class LiveServerCommand(Command):
#     """Run a server from within a cube directory.
#     """
#     name = 'live-server'
#     arguments = ''
#     options = ()
513

514
515
516
517
#     def run(self, args):
#         """run the command with its specific arguments"""
#         from cubicweb.devtools.livetest import runserver
#         runserver()
Adrien Di Mascio's avatar
Adrien Di Mascio committed
518
519


520
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
521
522
523
    """Create a new cube.

    <cubename>
524
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
525
526
527
    """
    name = 'newcube'
    arguments = '<cubename>'
528
    min_args = max_args = 1
529
    options = (
530
531
532
533
534
535
536
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
537
538
539
540
541
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
542
543
544
545
546
547
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
566
567
568
569
570
571
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
572
573
        )

574
    LICENSES = {
575
576
577
578
579
580
581
582
583
584
585
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
586
587
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
588
''',
589

590
591
592
593
594
595
596
597
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
598
599
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
600
601
602
603
604
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
605
        }
606

Adrien Di Mascio's avatar
Adrien Di Mascio committed
607
    def run(self, args):
608
        import re
609
        from logilab.common.shellutils import ASK
610
611
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
612
613
            raise BadCommandUsage(
                'cube name must be a valid python module name')
614
        verbose = self.get('verbose')
615
        cubesdir = self.get('directory')
616
617
        if not cubesdir:
            cubespath = ServerConfiguration.cubes_search_path()
618
            if len(cubespath) > 1:
619
620
621
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
622
            cubesdir = cubespath[0]
623
        if not osp.isdir(cubesdir):
Samuel Trégouët's avatar
Samuel Trégouët committed
624
            print("-> creating cubes directory", cubesdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
625
            try:
626
                mkdir(cubesdir)
627
            except OSError as err:
628
629
630
631
                self.fail("failed to create directory %r\n(%s)"
                          % (cubesdir, err))
        cubedir = osp.join(cubesdir, cubename)
        if osp.exists(cubedir):
Dimitri Papadopoulos's avatar
Dimitri Papadopoulos committed
632
            self.fail("%s already exists!" % cubedir)
633
        skeldir = osp.join(BASEDIR, 'skeleton')
634
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
635
        if verbose:
636
637
            distname = raw_input('Debian name for your cube ? [%s]): '
                                 % default_name).strip()
638
            if not distname:
639
                distname = default_name
640
            elif not distname.startswith('cubicweb-'):
641
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
642
643
                    distname = 'cubicweb-' + distname
        else:
644
            distname = default_name
645
        if not re.match('[a-z][-a-z0-9]*$', distname):
646
647
648
649
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
        longdesc = shortdesc = raw_input(
            'Enter a short description for your cube: ')
650
        if verbose:
651
652
            longdesc = raw_input(
                'Enter a long description (leave empty to reuse the short one): ')
653
654
        dependencies = {'cubicweb': '>= %s' % cubicwebversion,
                        'six': '>= 1.4.0',}
655
        if verbose:
656
            dependencies.update(self._ask_for_dependencies())
Adrien Di Mascio's avatar
Adrien Di Mascio committed
657
658
659
660
        context = {'cubename' : cubename,
                   'distname' : distname,
                   'shortdesc' : shortdesc,
                   'longdesc' : longdesc or shortdesc,
661
                   'dependencies' : dependencies,
Adrien Di Mascio's avatar
Adrien Di Mascio committed
662
                   'version'  : cubicwebversion,
663
                   'year'  : str(date.today().year),
664
665
666
                   'author': self['author'],
                   'author-email': self['author-email'],
                   'author-web-site': self['author-web-site'],
667
668
                   'license': self['license'],
                   'long-license': self.LICENSES[self['license']],
Adrien Di Mascio's avatar
Adrien Di Mascio committed
669
                   }
670
671
672
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
673
                        'cubes.*', 'uiprops.py*')
674
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
675

676
    def _ask_for_dependencies(self):
677
678
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
679
680
681
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
682
683
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
684
                depcubes.append(cube)
685
            if answer == 'type':
686
                depcubes = splitstrip(raw_input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
687
                break
688
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
689
                break
690
691
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
692

693
694
695
696

class ExamineLogCommand(Command):
    """Examine a rql log file.

697
    Will print out the following table
698

699
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
700

701
    sorted by descending cumulative time (clock). Time are expressed in seconds.
702

703
    Chances are the lines at the top are the ones that will bring the higher
704
    benefit after optimisation. Start there.
705
    """
706
    arguments = 'rql.log'
707
    name = 'exlog'
708
    options = ()
709

710
711
712
    def run(self, args):
        import re
        requests = {}
713
        for filepath in args:
714
            try:
Rémi Cardona's avatar
Rémi Cardona committed
715
                stream = open(filepath)
716
            except OSError as ex:
717
718
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
719
            for lineno, line in enumerate(stream):
720
721
722
723
724
725
726
727
728
729
730
731
732
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
733
                except Exception as exc:
734
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
735
        stat = []
736
        for rql, times in requests.items():
737
738
739
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
740
741
        stat.sort()
        stat.reverse()
742
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
743
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
744
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
745
746
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
747

748

749
750
751
752
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
753
    min_args = max_args = 1
754
755
    options = [
        ('output-file',
756
         {'type':'string', 'default': None,
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
790
791
792
793
794
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
795
        ]
796
797

    def run(self, args):
798
799
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
800
        from logilab.common.textutils import splitstrip
801
802
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
803
804
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
805
        cubes = splitstrip(args[0])
806
        dev_conf = DevConfiguration(*cubes)
807
808
809
810
811
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
812
813
814
815
816
817
818
819
820
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

837
838
839
840
        if viewer:
            p = Popen((viewer, out))
            p.wait()

841

842
for cmdcls in (UpdateCubicWebCatalogCommand,
843
               UpdateCubeCatalogCommand,
844
845
846
847
848
849
               #LiveServerCommand,
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)