devctl.py 36.1 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

23
24
25
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
26
27
28

import shutil
import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
29
import sys
30
from datetime import datetime, date
31
from os import mkdir, chdir, path as osp
32
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
33

34
from pytz import UTC
35
36
from six.moves import input

Adrien Di Mascio's avatar
Adrien Di Mascio committed
37
from logilab.common import STD_BLACKLIST
38
39
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find
40

Adrien Di Mascio's avatar
Adrien Di Mascio committed
41
from cubicweb.__pkginfo__ import version as cubicwebversion
42
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
43
from cubicweb.i18n import extract_from_tal, execute2
44
from cubicweb.cwctl import CWCTL
45
from cubicweb.cwconfig import CubicWebNoAppConfiguration
46
47
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
48
49
50
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

51

52
53
54
__docformat__ = "restructuredtext en"


55
56
57
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')
58
STD_BLACKLIST.add('node_modules')
59
60


61
62
63
64
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
65
    creating = True
66
    cleanup_unused_appobjects = False
67
68
69
70
71

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
72

73
    def __init__(self, *cubes):
74
75
76
77
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
78
            self.load_site_cubicweb()
79
80
        else:
            self._cubes = ()
81

Adrien Di Mascio's avatar
Adrien Di Mascio committed
82
83
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
84
        return None
85
86
87
88

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
89
90
    def main_config_file(self):
        return None
91
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
92
        pass
93
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
94
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
95
96
    def default_log_file(self):
        return None
97
98
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
99
100
101
102


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
103
    for name, mod in list(sys.modules.items()):
Sylvain Thenault's avatar
Sylvain Thenault committed
104
105
106
107
108
109
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
110
111
112
        if mod.__file__ is None:
            # odd/rare but real
            continue
113
        for path in config.appobjects_path():
Sylvain Thenault's avatar
Sylvain Thenault committed
114
115
116
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
117

Adrien Di Mascio's avatar
Adrien Di Mascio committed
118
119
120
121
122
123
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
124
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
125
    if cubedir:
126
        cube = osp.split(cubedir)[-1]
127
128
        if cube.startswith('cubicweb_'):
            cube = cube[len('cubicweb_'):]
129
130
131
132
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
133
    else:
134
135
136
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
137
    schema = config.load_schema(remove_unused_rtypes=False)
138
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
139
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
140
141
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
142
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
143
144


145
def _generate_schema_pot(w, vreg, schema, libconfig=None):
146
    from cubicweb.i18n import add_msg
147
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
148
    w('# schema pot file, generated on %s\n'
149
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
150
151
152
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
153
    vregdone = set()
154
    afss = vreg['uicfg']['autoform_section']
155
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
156
    if libconfig is not None:
157
158
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
159
        from cubicweb.cwvreg import CWRegistryStore
160
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
161
        cleanup_sys_modules(libconfig)
162
        libvreg = CWRegistryStore(libconfig)
163
        libvreg.set_schema(libschema) # trigger objects registration
164
        libafss = libvreg['uicfg']['autoform_section']
165
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
166
167
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
168
169
170
171

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
172
    else:
173
        # processing cubicweb itself
174
        libschema = {}
175
176
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
177
178
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
179
    done = set()
180
181
182
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
183
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
184
        etype = eschema.type
185
186
187
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
188
            if not eschema.final:
189
                add_msg(w, 'This %s:' % etype)
190
                add_msg(w, 'New %s' % etype)
191
                add_msg(w, 'add a %s' % etype) # AddNewAction
192
193
194
195
196
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
197
198
199
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
200
        if eschema.final:
201
202
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
203
204
            if rschema.final:
                continue
205
            for tschema in targetschemas:
206

207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
234
235
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
236
    w('# subject and object forms for each relation type\n')
237
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
238
    w('\n')
239
240
241
242
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
243
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
244
        rtype = rschema.type
245
        if rtype not in libschema:
246
247
            # bw compat, necessary until all translation of relation are done
            # properly...
248
            add_msg(w, rtype)
249
            done.add(rtype)
250
251
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
252
            done.add(rschema.description)
253
254
255
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
256
        # add context information only for non-metadata rtypes
257
        if rschema not in NO_I18NCONTEXT:
258
            libsubjects = librschema and librschema.subjects() or ()
259
            for subjschema in rschema.subjects():
260
261
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
262
        if not (rschema.final or rschema.symmetric):
263
            if rschema not in NO_I18NCONTEXT:
264
265
266
267
268
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
269
270
                # bw compat, necessary until all translation of relation are
                # done properly...
271
                add_msg(w, '%s_object' % rtype)
272
        for rdef in rschema.rdefs.values():
273
274
275
276
277
278
279
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
280
    for objid in _iter_vreg_objids(vreg, vregdone):
281
282
283
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
284

285
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
286
    for reg, objdict in vreg.items():
287
288
        if reg in ('boxes', 'contentnavigation'):
            continue
289
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
290
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
291
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
292
                if objid in done:
293
                    break
294
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
295
                if pdefs:
296
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
297
                    done.add(objid)
298
                    break
299

300

Adrien Di Mascio's avatar
Adrien Di Mascio committed
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
318

Adrien Di Mascio's avatar
Adrien Di Mascio committed
319
320
321
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
322
    name = 'i18ncubicweb'
323
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
324
325
326
327

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
328
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
329
330
        import yams
        from logilab.common.fileutils import ensure_fs_mode
331
        from logilab.common.shellutils import globfind, find, rm
332
        from logilab.common.modutils import get_module_files
333
        from cubicweb.i18n import extract_from_tal, execute2
334
        tempdir = tempfile.mkdtemp(prefix='cw-')
335
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
336
337
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
338
339
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
340
341
342
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
343
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
344
345
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
346
        print('TAL', end=' ')
347
348
349
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
350
        print('-> generate .pot files.')
351
352
353
354
355
356
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
357
358
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
359
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
360
                                ]:
361
            potfile = osp.join(tempdir, '%s.pot' % id)
362
363
364
365
366
367
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
368
            if osp.exists(potfile):
369
370
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
371
372
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
373
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
374
375
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
376
        print('-> merging main pot file with existing translations.')
377
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
378
        toedit = []
379
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
380
            target = '%s.po' % lang
381
382
383
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
384
385
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
386
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
387
        # cleanup
388
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
389
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
390
391
392
393
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
394
395


396
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
397
398
399
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
400
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
401
    arguments = '[<cube>...]'
402

Adrien Di Mascio's avatar
Adrien Di Mascio committed
403
404
405
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
406
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
407
        else:
408
409
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
410
411
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
412
413
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
414

415

Adrien Di Mascio's avatar
Adrien Di Mascio committed
416
def update_cubes_catalogs(cubes):
417
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
418
    for cubedir in cubes:
419
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
420
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
421
            continue
422
        try:
423
            toedit = update_cube_catalogs(cubedir)
424
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
425
426
427
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
428
429
430
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
431
            print('*** error while updating catalogs for cube', cubedir)
432
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
433
434
        else:
            # instructions pour la suite
435
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
436
437
438
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
439
440
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
441
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
442

443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495

class I18nCubeMessageExtractor(object):
    """This class encapsulates all the xgettext extraction logic

    ``generate_pot_file`` is the main entry point called by the ``i18ncube``
    command. A cube might decide to customize extractors to ignore a given
    directory or to extract messages from a new file type (e.g. .jinja2 files)

    For each file type, the class must define two methods:

    - ``collect_{filetype}()`` that must return the list of files
      xgettext should inspect,

    - ``extract_{filetype}(files)`` that calls xgettext and returns the
      path to the generated ``pot`` file
    """
    blacklist = STD_BLACKLIST
    formats = ['tal', 'js', 'py']

    def __init__(self, workdir, cubedir):
        self.workdir = workdir
        self.cubedir = cubedir

    def generate_pot_file(self):
        """main entry point: return the generated ``cube.pot`` file

        This function first generates all the pot files (schema, tal,
        py, js) and then merges them in a single ``cube.pot`` that will
        be used to eventually update the ``i18n/*.po`` files.
        """
        potfiles = self.generate_pot_files()
        potfile = osp.join(self.workdir, 'cube.pot')
        print('-> merging %i .pot files' % len(potfiles))
        cmd = ['msgcat', '-o', potfile]
        cmd.extend(potfiles)
        execute2(cmd)
        return potfile if osp.exists(potfile) else None

    def find(self, exts, blacklist=None):
        """collect files with extensions ``exts`` in the cube directory
        """
        if blacklist is None:
            blacklist = self.blacklist
        return find(self.cubedir, exts, blacklist=blacklist)

    def generate_pot_files(self):
        """generate and return the list of all ``pot`` files for the cube

        - static-messages.pot,
        - schema.pot,
        - one ``pot`` file for each inspected format (.py, .js, etc.)
        """
        print('-> extracting messages:', end=' ')
496
        potfiles = []
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
        # static messages
        if osp.exists(osp.join('i18n', 'entities.pot')):
            warn('entities.pot is deprecated, rename file '
                 'to static-messages.pot (%s)'
                 % osp.join('i18n', 'entities.pot'), DeprecationWarning)
            potfiles.append(osp.join('i18n', 'entities.pot'))
        elif osp.exists(osp.join('i18n', 'static-messages.pot')):
            potfiles.append(osp.join('i18n', 'static-messages.pot'))
        # messages from schema
        potfiles.append(self.schemapot())
        # messages from sourcecode
        for fmt in self.formats:
            collector = getattr(self, 'collect_{0}'.format(fmt))
            extractor = getattr(self, 'extract_{0}'.format(fmt))
            files = collector()
            if files:
                potfile = extractor(files)
                if potfile:
                    potfiles.append(potfile)
        return potfiles

    def schemapot(self):
        """generate the ``schema.pot`` file"""
        schemapot = osp.join(self.workdir, 'schema.pot')
        print('schema', end=' ')
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
        schemapotstream = open(schemapot, 'w')
        generate_schema_pot(schemapotstream.write, self.cubedir)
        schemapotstream.close()
        return schemapot

    def _xgettext(self, files, output, k='_', extraopts=''):
        """shortcut to execute the xgettext command and return output file
        """
        tmppotfile = osp.join(self.workdir, output)
        cmd = ['xgettext', '--no-location', '--omit-header', '-k' + k,
               '-o', tmppotfile] + extraopts.split() + files
535
        execute2(cmd)
536
        if osp.exists(tmppotfile):
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
            return tmppotfile

    def collect_tal(self):
        print('TAL', end=' ')
        return self.find(('.py', '.pt'))

    def extract_tal(self, files):
        tali18nfile = osp.join(self.workdir, 'tali18n.py')
        extract_from_tal(files, tali18nfile)
        return self._xgettext(files, output='tal.pot')

    def collect_js(self):
        print('Javascript')
        return [jsfile for jsfile in self.find('.js')
                if osp.basename(jsfile).startswith('cub')]

    def extract_js(self, files):
        return self._xgettext(files, output='js.pot',
                              extraopts='-L java --from-code=utf-8')

    def collect_py(self):
        print('-> creating cube-specific catalog')
        return self.find('.py')

    def extract_py(self, files):
        return self._xgettext(files, output='py.pot')


def update_cube_catalogs(cubedir):
    cubedir = osp.abspath(osp.normpath(cubedir))
    workdir = tempfile.mkdtemp()
    cube = osp.basename(cubedir)
    print('cubedir', cubedir)
    print(underline_title('Updating i18n catalogs for cube %s' % cube))
    chdir(cubedir)
    extractor = I18nCubeMessageExtractor(workdir, cubedir)
    potfile = extractor.generate_pot_file()
    if potfile is None:
Samuel Trégouët's avatar
Samuel Trégouët committed
575
        print('no message catalog for cube', cube, 'nothing to translate')
576
        shutil.rmtree(workdir)
577
        return ()
Samuel Trégouët's avatar
Samuel Trégouët committed
578
    print('-> merging main pot file with existing translations:', end=' ')
579
    chdir('i18n')
580
    toedit = []
581
    for lang in CubicWebNoAppConfiguration.cw_languages():
Samuel Trégouët's avatar
Samuel Trégouët committed
582
        print(lang, end=' ')
583
        cubepo = '%s.po' % lang
584
        if not osp.exists(cubepo):
585
586
            shutil.copy(potfile, cubepo)
        else:
587
588
            cmd = ['msgmerge', '-N', '-s', '-o', cubepo + 'new',
                   cubepo, potfile]
589
            execute2(cmd)
590
591
            ensure_fs_mode(cubepo)
            shutil.move('%snew' % cubepo, cubepo)
592
        toedit.append(osp.abspath(cubepo))
Samuel Trégouët's avatar
Samuel Trégouët committed
593
    print()
594
    # cleanup
595
    shutil.rmtree(workdir)
596
597
598
    return toedit


599
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
600
601
602
    """Create a new cube.

    <cubename>
603
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
604
605
606
    """
    name = 'newcube'
    arguments = '<cubename>'
607
    min_args = max_args = 1
608
    options = (
609
610
611
612
613
614
615
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
616
617
618
619
620
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
621
622
623
624
625
626
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
645
646
647
648
649
650
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
651
652
        )

653
    LICENSES = {
654
655
656
657
658
659
660
661
662
663
664
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
665
666
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
667
''',
668

669
670
671
672
673
674
675
676
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
677
678
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
679
680
681
682
683
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
684
        }
685

Adrien Di Mascio's avatar
Adrien Di Mascio committed
686
    def run(self, args):
687
        import re
688
        from logilab.common.shellutils import ASK
689
690
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
691
692
            raise BadCommandUsage(
                'cube name must be a valid python module name')
693
        verbose = self.get('verbose')
694
695
        destdir = self.get('directory')
        if not destdir:
696
            cubespath = ServerConfiguration.cubes_search_path()
697
            if len(cubespath) > 1:
698
699
700
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
701
702
703
            destdir = cubespath[0]
        if not osp.isdir(destdir):
            print("-> creating cubes directory", destdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
704
            try:
705
                mkdir(destdir)
706
            except OSError as err:
707
                self.fail("failed to create directory %r\n(%s)"
708
                          % (destdir, err))
709
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
710
        if verbose:
711
            distname = input('Debian name for your cube ? [%s]): '
712
                             % default_name).strip()
713
            if not distname:
714
                distname = default_name
715
            elif not distname.startswith('cubicweb-'):
716
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
717
718
                    distname = 'cubicweb-' + distname
        else:
719
            distname = default_name
720
        if not re.match('[a-z][-a-z0-9]*$', distname):
721
722
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
723
724
725
726
        cubedir = osp.join(destdir, distname)
        if osp.exists(cubedir):
            self.fail("%s already exists!" % cubedir)
        skeldir = osp.join(BASEDIR, 'skeleton')
727
        longdesc = shortdesc = input(
728
            'Enter a short description for your cube: ')
729
        if verbose:
730
            longdesc = input(
731
                'Enter a long description (leave empty to reuse the short one): ')
732
733
734
735
        dependencies = {
            'six': '>= 1.4.0',
            'cubicweb': '>= %s' % cubicwebversion,
        }
736
        if verbose:
737
            dependencies.update(self._ask_for_dependencies())
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
        context = {
            'cubename': cubename,
            'distname': distname,
            'shortdesc': shortdesc,
            'longdesc': longdesc or shortdesc,
            'dependencies': dependencies,
            'version': cubicwebversion,
            'year': str(date.today().year),
            'author': self['author'],
            'author-email': self['author-email'],
            'rfc2822-date': datetime.now(tz=UTC).strftime('%a, %d %b %Y %T %z'),
            'author-web-site': self['author-web-site'],
            'license': self['license'],
            'long-license': self.LICENSES[self['license']],
        }
753
754
755
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
756
                        'cubes.*', 'uiprops.py*')
757
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
758

759
    def _ask_for_dependencies(self):
760
761
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
762
763
764
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
765
766
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
767
                depcubes.append(cube)
768
            if answer == 'type':
769
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
770
                break
771
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
772
                break
773
774
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
775

776
777
778
779

class ExamineLogCommand(Command):
    """Examine a rql log file.

780
    Will print out the following table
781

782
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
783

784
    sorted by descending cumulative time (clock). Time are expressed in seconds.
785

786
    Chances are the lines at the top are the ones that will bring the higher
787
    benefit after optimisation. Start there.
788
    """
789
    arguments = 'rql.log'
790
    name = 'exlog'
791
    options = ()
792

793
794
795
    def run(self, args):
        import re
        requests = {}
796
        for filepath in args:
797
            try:
Rémi Cardona's avatar
Rémi Cardona committed
798
                stream = open(filepath)
799
            except OSError as ex:
800
801
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
802
            for lineno, line in enumerate(stream):
803
804
805
806
807
808
809
810
811
812
813
814
815
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
816
                except Exception as exc:
817
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
818
        stat = []
819
        for rql, times in requests.items():
820
821
822
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
823
824
        stat.sort()
        stat.reverse()
825
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
826
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
827
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
828
829
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
830

831

832
833
834
835
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
836
    min_args = max_args = 1
837
838
    options = [
        ('output-file',
839
         {'type':'string', 'default': None,
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
873
874
875
876
877
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
878
        ]
879
880

    def run(self, args):
881
882
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
883
        from logilab.common.textutils import splitstrip
884
885
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
886
887
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
888
        cubes = splitstrip(args[0])
889
        dev_conf = DevConfiguration(*cubes)
890
891
892
893
894
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
895
896
897
898
899
900
901
902
903
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

920
921
922
923
        if viewer:
            p = Popen((viewer, out))
            p.wait()

924

925
for cmdcls in (UpdateCubicWebCatalogCommand,
926
               UpdateCubeCatalogCommand,
927
928
929
930
931
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)