devctl.py 36.6 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

23
24
25
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
26
27
28

import shutil
import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
29
import sys
30
from datetime import datetime, date
31
from os import mkdir, chdir, path as osp
32
import pkg_resources
33
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
34

35
from pytz import UTC
36

37
38
from six.moves import input

Adrien Di Mascio's avatar
Adrien Di Mascio committed
39
from logilab.common import STD_BLACKLIST
40
41
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find
42

Adrien Di Mascio's avatar
Adrien Di Mascio committed
43
from cubicweb.__pkginfo__ import version as cubicwebversion
44
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
45
from cubicweb.i18n import extract_from_tal, execute2
46
from cubicweb.cwctl import CWCTL
47
from cubicweb.cwconfig import CubicWebNoAppConfiguration
48
49
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
50
51
52
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

53

54
55
56
__docformat__ = "restructuredtext en"


57
58
59
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')
60
STD_BLACKLIST.add('node_modules')
61
62


63
64
65
66
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
67
    creating = True
68
    cleanup_unused_appobjects = False
69
70
71
72
73

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
74

75
    def __init__(self, *cubes):
76
77
78
79
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
80
            self.load_site_cubicweb()
81
82
        else:
            self._cubes = ()
83

Adrien Di Mascio's avatar
Adrien Di Mascio committed
84
85
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
86
        return None
87
88
89
90

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
91
92
    def main_config_file(self):
        return None
93
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
94
        pass
95
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
96
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
97
98
    def default_log_file(self):
        return None
99
100
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
101
102
103
104


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
105
    for name, mod in list(sys.modules.items()):
Sylvain Thenault's avatar
Sylvain Thenault committed
106
107
108
109
110
111
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
112
113
114
        if mod.__file__ is None:
            # odd/rare but real
            continue
115
        for path in config.appobjects_path():
Sylvain Thenault's avatar
Sylvain Thenault committed
116
117
118
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
119

Adrien Di Mascio's avatar
Adrien Di Mascio committed
120
121
122
123
124
125
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
126
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
127
    if cubedir:
128
        cube = osp.split(cubedir)[-1]
129
130
        if cube.startswith('cubicweb_'):
            cube = cube[len('cubicweb_'):]
131
132
133
134
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
135
    else:
136
137
138
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
139
    schema = config.load_schema(remove_unused_rtypes=False)
140
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
141
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
142
143
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
144
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
145
146


147
def _generate_schema_pot(w, vreg, schema, libconfig=None):
148
    from cubicweb.i18n import add_msg
149
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
150
    w('# schema pot file, generated on %s\n'
151
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
152
153
154
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
155
    vregdone = set()
156
    afss = vreg['uicfg']['autoform_section']
157
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
158
    if libconfig is not None:
159
160
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
161
        from cubicweb.cwvreg import CWRegistryStore
162
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
163
        cleanup_sys_modules(libconfig)
164
        libvreg = CWRegistryStore(libconfig)
165
        libvreg.set_schema(libschema) # trigger objects registration
166
        libafss = libvreg['uicfg']['autoform_section']
167
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
168
169
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
170
171
172
173

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
174
    else:
175
        # processing cubicweb itself
176
        libschema = {}
177
178
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
179
180
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
181
    done = set()
182
183
184
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
185
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
186
        etype = eschema.type
187
188
189
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
190
            if not eschema.final:
191
                add_msg(w, 'This %s:' % etype)
192
                add_msg(w, 'New %s' % etype)
193
                add_msg(w, 'add a %s' % etype) # AddNewAction
194
195
196
197
198
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
199
200
201
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
202
        if eschema.final:
203
204
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
205
206
            if rschema.final:
                continue
207
            for tschema in targetschemas:
208

209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
236
237
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
238
    w('# subject and object forms for each relation type\n')
239
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
240
    w('\n')
241
242
243
244
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
245
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
246
        rtype = rschema.type
247
        if rtype not in libschema:
248
249
            # bw compat, necessary until all translation of relation are done
            # properly...
250
            add_msg(w, rtype)
251
            done.add(rtype)
252
253
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
254
            done.add(rschema.description)
255
256
257
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
258
        # add context information only for non-metadata rtypes
259
        if rschema not in NO_I18NCONTEXT:
260
            libsubjects = librschema and librschema.subjects() or ()
261
            for subjschema in rschema.subjects():
262
263
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
264
        if not (rschema.final or rschema.symmetric):
265
            if rschema not in NO_I18NCONTEXT:
266
267
268
269
270
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
271
272
                # bw compat, necessary until all translation of relation are
                # done properly...
273
                add_msg(w, '%s_object' % rtype)
274
        for rdef in rschema.rdefs.values():
275
276
277
278
279
280
281
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
282
    for objid in _iter_vreg_objids(vreg, vregdone):
283
284
285
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
286

287
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
288
    for reg, objdict in vreg.items():
289
290
        if reg in ('boxes', 'contentnavigation'):
            continue
291
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
292
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
293
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
294
                if objid in done:
295
                    break
296
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
297
                if pdefs:
298
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
299
                    done.add(objid)
300
                    break
301

302

Adrien Di Mascio's avatar
Adrien Di Mascio committed
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
320

Adrien Di Mascio's avatar
Adrien Di Mascio committed
321
322
323
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
324
    name = 'i18ncubicweb'
325
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
326
327
328
329

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
330
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
331
332
        import yams
        from logilab.common.fileutils import ensure_fs_mode
333
        from logilab.common.shellutils import globfind, find, rm
334
        from logilab.common.modutils import get_module_files
335
        from cubicweb.i18n import extract_from_tal, execute2
336
        tempdir = tempfile.mkdtemp(prefix='cw-')
337
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
338
339
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
340
341
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
342
343
344
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
345
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
346
347
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
348
        print('TAL', end=' ')
349
350
351
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
352
        print('-> generate .pot files.')
353
354
355
356
357
358
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
359
360
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
361
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
362
                                ]:
363
            potfile = osp.join(tempdir, '%s.pot' % id)
364
365
366
367
368
369
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
370
            if osp.exists(potfile):
371
372
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
373
374
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
375
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
376
377
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
378
        print('-> merging main pot file with existing translations.')
379
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
380
        toedit = []
381
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
382
            target = '%s.po' % lang
383
384
385
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
386
387
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
388
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
389
        # cleanup
390
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
391
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
392
393
394
395
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
396
397


398
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
399
400
401
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
402
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
403
    arguments = '[<cube>...]'
404

Adrien Di Mascio's avatar
Adrien Di Mascio committed
405
406
407
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
408
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
409
        else:
410
411
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
412
413
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
414
415
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
416

417

Adrien Di Mascio's avatar
Adrien Di Mascio committed
418
def update_cubes_catalogs(cubes):
419
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
420
    for cubedir in cubes:
421
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
422
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
423
            continue
424
        try:
425
            toedit = update_cube_catalogs(cubedir)
426
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
427
428
429
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
430
431
432
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
433
            print('*** error while updating catalogs for cube', cubedir)
434
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
435
436
        else:
            # instructions pour la suite
437
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
438
439
440
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
441
442
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
443
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
444

445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497

class I18nCubeMessageExtractor(object):
    """This class encapsulates all the xgettext extraction logic

    ``generate_pot_file`` is the main entry point called by the ``i18ncube``
    command. A cube might decide to customize extractors to ignore a given
    directory or to extract messages from a new file type (e.g. .jinja2 files)

    For each file type, the class must define two methods:

    - ``collect_{filetype}()`` that must return the list of files
      xgettext should inspect,

    - ``extract_{filetype}(files)`` that calls xgettext and returns the
      path to the generated ``pot`` file
    """
    blacklist = STD_BLACKLIST
    formats = ['tal', 'js', 'py']

    def __init__(self, workdir, cubedir):
        self.workdir = workdir
        self.cubedir = cubedir

    def generate_pot_file(self):
        """main entry point: return the generated ``cube.pot`` file

        This function first generates all the pot files (schema, tal,
        py, js) and then merges them in a single ``cube.pot`` that will
        be used to eventually update the ``i18n/*.po`` files.
        """
        potfiles = self.generate_pot_files()
        potfile = osp.join(self.workdir, 'cube.pot')
        print('-> merging %i .pot files' % len(potfiles))
        cmd = ['msgcat', '-o', potfile]
        cmd.extend(potfiles)
        execute2(cmd)
        return potfile if osp.exists(potfile) else None

    def find(self, exts, blacklist=None):
        """collect files with extensions ``exts`` in the cube directory
        """
        if blacklist is None:
            blacklist = self.blacklist
        return find(self.cubedir, exts, blacklist=blacklist)

    def generate_pot_files(self):
        """generate and return the list of all ``pot`` files for the cube

        - static-messages.pot,
        - schema.pot,
        - one ``pot`` file for each inspected format (.py, .js, etc.)
        """
        print('-> extracting messages:', end=' ')
498
        potfiles = []
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
        # static messages
        if osp.exists(osp.join('i18n', 'entities.pot')):
            warn('entities.pot is deprecated, rename file '
                 'to static-messages.pot (%s)'
                 % osp.join('i18n', 'entities.pot'), DeprecationWarning)
            potfiles.append(osp.join('i18n', 'entities.pot'))
        elif osp.exists(osp.join('i18n', 'static-messages.pot')):
            potfiles.append(osp.join('i18n', 'static-messages.pot'))
        # messages from schema
        potfiles.append(self.schemapot())
        # messages from sourcecode
        for fmt in self.formats:
            collector = getattr(self, 'collect_{0}'.format(fmt))
            extractor = getattr(self, 'extract_{0}'.format(fmt))
            files = collector()
            if files:
                potfile = extractor(files)
                if potfile:
                    potfiles.append(potfile)
        return potfiles

    def schemapot(self):
        """generate the ``schema.pot`` file"""
        schemapot = osp.join(self.workdir, 'schema.pot')
        print('schema', end=' ')
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
        schemapotstream = open(schemapot, 'w')
        generate_schema_pot(schemapotstream.write, self.cubedir)
        schemapotstream.close()
        return schemapot

    def _xgettext(self, files, output, k='_', extraopts=''):
        """shortcut to execute the xgettext command and return output file
        """
        tmppotfile = osp.join(self.workdir, output)
        cmd = ['xgettext', '--no-location', '--omit-header', '-k' + k,
               '-o', tmppotfile] + extraopts.split() + files
537
        execute2(cmd)
538
        if osp.exists(tmppotfile):
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
            return tmppotfile

    def collect_tal(self):
        print('TAL', end=' ')
        return self.find(('.py', '.pt'))

    def extract_tal(self, files):
        tali18nfile = osp.join(self.workdir, 'tali18n.py')
        extract_from_tal(files, tali18nfile)
        return self._xgettext(files, output='tal.pot')

    def collect_js(self):
        print('Javascript')
        return [jsfile for jsfile in self.find('.js')
                if osp.basename(jsfile).startswith('cub')]

    def extract_js(self, files):
        return self._xgettext(files, output='js.pot',
                              extraopts='-L java --from-code=utf-8')

    def collect_py(self):
        print('-> creating cube-specific catalog')
        return self.find('.py')

    def extract_py(self, files):
        return self._xgettext(files, output='py.pot')


def update_cube_catalogs(cubedir):
    cubedir = osp.abspath(osp.normpath(cubedir))
    workdir = tempfile.mkdtemp()
570
    try:
571
572
        distname = osp.basename(cubedir)
        cubename = distname.split('_')[-1]
573
        print('cubedir', cubedir)
574
575
576
577
578
579
580
        extract_cls = I18nCubeMessageExtractor
        try:
            extract_cls = pkg_resources.load_entry_point(
                distname, 'cubicweb.i18ncube', cubename)
        except (pkg_resources.DistributionNotFound, ImportError):
            pass  # no customization found
        print(underline_title('Updating i18n catalogs for cube %s' % cubename))
581
        chdir(cubedir)
582
        extractor = extract_cls(workdir, cubedir)
583
584
        potfile = extractor.generate_pot_file()
        if potfile is None:
585
            print('no message catalog for cube', cubename, 'nothing to translate')
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
            return ()
        print('-> merging main pot file with existing translations:', end=' ')
        chdir('i18n')
        toedit = []
        for lang in CubicWebNoAppConfiguration.cw_languages():
            print(lang, end=' ')
            cubepo = '%s.po' % lang
            if not osp.exists(cubepo):
                shutil.copy(potfile, cubepo)
            else:
                cmd = ['msgmerge', '-N', '-s', '-o', cubepo + 'new',
                       cubepo, potfile]
                execute2(cmd)
                ensure_fs_mode(cubepo)
                shutil.move('%snew' % cubepo, cubepo)
            toedit.append(osp.abspath(cubepo))
        print()
        return toedit
    finally:
        # cleanup
606
        shutil.rmtree(workdir)
607
608


609
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
610
611
612
    """Create a new cube.

    <cubename>
613
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
614
615
616
    """
    name = 'newcube'
    arguments = '<cubename>'
617
    min_args = max_args = 1
618
    options = (
619
620
621
622
623
624
625
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
626
627
628
629
630
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
631
632
633
634
635
636
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
655
656
657
658
659
660
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
661
662
        )

663
    LICENSES = {
664
665
666
667
668
669
670
671
672
673
674
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
675
676
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
677
''',
678

679
680
681
682
683
684
685
686
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
687
688
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
689
690
691
692
693
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
694
        }
695

Adrien Di Mascio's avatar
Adrien Di Mascio committed
696
    def run(self, args):
697
        import re
698
        from logilab.common.shellutils import ASK
699
700
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
701
702
            raise BadCommandUsage(
                'cube name must be a valid python module name')
703
        verbose = self.get('verbose')
704
705
        destdir = self.get('directory')
        if not destdir:
706
            cubespath = ServerConfiguration.cubes_search_path()
707
            if len(cubespath) > 1:
708
709
710
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
711
712
713
            destdir = cubespath[0]
        if not osp.isdir(destdir):
            print("-> creating cubes directory", destdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
714
            try:
715
                mkdir(destdir)
716
            except OSError as err:
717
                self.fail("failed to create directory %r\n(%s)"
718
                          % (destdir, err))
719
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
720
        if verbose:
721
            distname = input('Debian name for your cube ? [%s]): '
722
                             % default_name).strip()
723
            if not distname:
724
                distname = default_name
725
            elif not distname.startswith('cubicweb-'):
726
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
727
728
                    distname = 'cubicweb-' + distname
        else:
729
            distname = default_name
730
        if not re.match('[a-z][-a-z0-9]*$', distname):
731
732
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
733
734
735
736
        cubedir = osp.join(destdir, distname)
        if osp.exists(cubedir):
            self.fail("%s already exists!" % cubedir)
        skeldir = osp.join(BASEDIR, 'skeleton')
737
        longdesc = shortdesc = input(
738
            'Enter a short description for your cube: ')
739
        if verbose:
740
            longdesc = input(
741
                'Enter a long description (leave empty to reuse the short one): ')
742
743
744
745
        dependencies = {
            'six': '>= 1.4.0',
            'cubicweb': '>= %s' % cubicwebversion,
        }
746
        if verbose:
747
            dependencies.update(self._ask_for_dependencies())
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
        context = {
            'cubename': cubename,
            'distname': distname,
            'shortdesc': shortdesc,
            'longdesc': longdesc or shortdesc,
            'dependencies': dependencies,
            'version': cubicwebversion,
            'year': str(date.today().year),
            'author': self['author'],
            'author-email': self['author-email'],
            'rfc2822-date': datetime.now(tz=UTC).strftime('%a, %d %b %Y %T %z'),
            'author-web-site': self['author-web-site'],
            'license': self['license'],
            'long-license': self.LICENSES[self['license']],
        }
763
764
765
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
766
                        'cubes.*', 'uiprops.py*')
767
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
768

769
    def _ask_for_dependencies(self):
770
771
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
772
773
774
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
775
776
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
777
                depcubes.append(cube)
778
            if answer == 'type':
779
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
780
                break
781
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
782
                break
783
784
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
785

786
787
788
789

class ExamineLogCommand(Command):
    """Examine a rql log file.

790
    Will print out the following table
791

792
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
793

794
    sorted by descending cumulative time (clock). Time are expressed in seconds.
795

796
    Chances are the lines at the top are the ones that will bring the higher
797
    benefit after optimisation. Start there.
798
    """
799
    arguments = 'rql.log'
800
    name = 'exlog'
801
    options = ()
802

803
804
805
    def run(self, args):
        import re
        requests = {}
806
        for filepath in args:
807
            try:
Rémi Cardona's avatar
Rémi Cardona committed
808
                stream = open(filepath)
809
            except OSError as ex:
810
811
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
812
            for lineno, line in enumerate(stream):
813
814
815
816
817
818
819
820
821
822
823
824
825
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
826
                except Exception as exc:
827
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
828
        stat = []
829
        for rql, times in requests.items():
830
831
832
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
833
834
        stat.sort()
        stat.reverse()
835
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
836
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
837
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
838
839
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
840

841

842
843
844
845
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
846
    min_args = max_args = 1
847
848
    options = [
        ('output-file',
849
         {'type':'string', 'default': None,
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
883
884
885
886
887
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
888
        ]
889
890

    def run(self, args):
891
892
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
893
        from logilab.common.textutils import splitstrip
894
895
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
896
897
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
898
        cubes = splitstrip(args[0])
899
        dev_conf = DevConfiguration(*cubes)
900
901
902
903
904
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
905
906
907
908
909
910
911
912
913
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

930
931
932
933
        if viewer:
            p = Popen((viewer, out))
            p.wait()

934

935
for cmdcls in (UpdateCubicWebCatalogCommand,
936
               UpdateCubeCatalogCommand,
937
938
939
940
941
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)