devctl.py 36.7 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

23
24
25
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
26
27
28

import shutil
import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
29
import sys
30
from datetime import datetime, date
31
from os import mkdir, chdir, path as osp
32
import pkg_resources
33
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
34

35
from pytz import UTC
36

37
38
from six.moves import input

Adrien Di Mascio's avatar
Adrien Di Mascio committed
39
from logilab.common import STD_BLACKLIST
40
41
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find
42

Adrien Di Mascio's avatar
Adrien Di Mascio committed
43
from cubicweb.__pkginfo__ import version as cubicwebversion
44
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
45
from cubicweb.i18n import extract_from_tal, execute2
46
from cubicweb.cwctl import CWCTL
47
from cubicweb.cwconfig import CubicWebNoAppConfiguration
48
49
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
50
51
52
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

53

54
55
56
__docformat__ = "restructuredtext en"


57
58
59
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')
60
STD_BLACKLIST.add('node_modules')
61
62


63
64
65
66
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
67
    creating = True
68
    cleanup_unused_appobjects = False
69
70
71
72
73

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
74

75
    def __init__(self, *cubes):
76
77
78
79
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
80
            self.load_site_cubicweb()
81
82
        else:
            self._cubes = ()
83

Adrien Di Mascio's avatar
Adrien Di Mascio committed
84
85
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
86
        return None
87
88
89
90

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
91
92
    def main_config_file(self):
        return None
93
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
94
        pass
95
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
96
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
97
98
    def default_log_file(self):
        return None
99
100
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
101
102
103
104


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
105
    appobjects_path = config.appobjects_path()
106
    for name, mod in list(sys.modules.items()):
Sylvain Thenault's avatar
Sylvain Thenault committed
107
108
109
110
111
112
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
113
114
115
        if mod.__file__ is None:
            # odd/rare but real
            continue
116
        for path in appobjects_path:
Sylvain Thenault's avatar
Sylvain Thenault committed
117
118
119
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
120

Adrien Di Mascio's avatar
Adrien Di Mascio committed
121
122
123
124
125
126
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
127
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
128
    if cubedir:
129
        cube = osp.split(cubedir)[-1]
130
131
        if cube.startswith('cubicweb_'):
            cube = cube[len('cubicweb_'):]
132
133
134
135
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
136
    else:
137
138
139
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
140
    schema = config.load_schema(remove_unused_rtypes=False)
141
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
142
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
143
144
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
145
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
146
147


148
def _generate_schema_pot(w, vreg, schema, libconfig=None):
149
    from cubicweb.i18n import add_msg
150
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
151
    w('# schema pot file, generated on %s\n'
152
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
153
154
155
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
156
    vregdone = set()
157
    afss = vreg['uicfg']['autoform_section']
158
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
159
    if libconfig is not None:
160
161
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
162
        from cubicweb.cwvreg import CWRegistryStore
163
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
164
        cleanup_sys_modules(libconfig)
165
        libvreg = CWRegistryStore(libconfig)
166
        libvreg.set_schema(libschema) # trigger objects registration
167
        libafss = libvreg['uicfg']['autoform_section']
168
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
169
170
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
171
172
173
174

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
175
    else:
176
        # processing cubicweb itself
177
        libschema = {}
178
179
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
180
181
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
182
    done = set()
183
184
185
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
186
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
187
        etype = eschema.type
188
189
190
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
191
            if not eschema.final:
192
                add_msg(w, 'This %s:' % etype)
193
                add_msg(w, 'New %s' % etype)
194
                add_msg(w, 'add a %s' % etype) # AddNewAction
195
196
197
198
199
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
200
201
202
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
203
        if eschema.final:
204
205
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
206
207
            if rschema.final:
                continue
208
            for tschema in targetschemas:
209

210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
237
238
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
239
    w('# subject and object forms for each relation type\n')
240
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
241
    w('\n')
242
243
244
245
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
246
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
247
        rtype = rschema.type
248
        if rtype not in libschema:
249
250
            # bw compat, necessary until all translation of relation are done
            # properly...
251
            add_msg(w, rtype)
252
            done.add(rtype)
253
254
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
255
            done.add(rschema.description)
256
257
258
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
259
        # add context information only for non-metadata rtypes
260
        if rschema not in NO_I18NCONTEXT:
261
            libsubjects = librschema and librschema.subjects() or ()
262
            for subjschema in rschema.subjects():
263
264
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
265
        if not (rschema.final or rschema.symmetric):
266
            if rschema not in NO_I18NCONTEXT:
267
268
269
270
271
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
272
273
                # bw compat, necessary until all translation of relation are
                # done properly...
274
                add_msg(w, '%s_object' % rtype)
275
        for rdef in rschema.rdefs.values():
276
277
278
279
280
281
282
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
283
    for objid in _iter_vreg_objids(vreg, vregdone):
284
285
286
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
287

288
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
289
    for reg, objdict in vreg.items():
290
291
        if reg in ('boxes', 'contentnavigation'):
            continue
292
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
293
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
294
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
295
                if objid in done:
296
                    break
297
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
298
                if pdefs:
299
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
300
                    done.add(objid)
301
                    break
302

303

Adrien Di Mascio's avatar
Adrien Di Mascio committed
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
321

Adrien Di Mascio's avatar
Adrien Di Mascio committed
322
323
324
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
325
    name = 'i18ncubicweb'
326
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
327
328
329
330

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
331
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
332
333
        import yams
        from logilab.common.fileutils import ensure_fs_mode
334
        from logilab.common.shellutils import globfind, find, rm
335
        from logilab.common.modutils import get_module_files
336
        from cubicweb.i18n import extract_from_tal, execute2
337
        tempdir = tempfile.mkdtemp(prefix='cw-')
338
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
339
340
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
341
342
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
343
344
345
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
346
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
347
348
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
349
        print('TAL', end=' ')
350
351
352
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
353
        print('-> generate .pot files.')
354
355
356
357
358
359
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
360
361
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
362
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
363
                                ]:
364
            potfile = osp.join(tempdir, '%s.pot' % id)
365
366
367
368
369
370
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
371
            if osp.exists(potfile):
372
373
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
374
375
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
376
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
377
378
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
379
        print('-> merging main pot file with existing translations.')
380
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
381
        toedit = []
382
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
383
            target = '%s.po' % lang
384
385
386
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
387
388
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
389
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
390
        # cleanup
391
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
392
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
393
394
395
396
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
397
398


399
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
400
401
402
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
403
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
404
    arguments = '[<cube>...]'
405

Adrien Di Mascio's avatar
Adrien Di Mascio committed
406
407
408
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
409
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
410
        else:
411
412
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
413
414
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
415
416
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
417

418

Adrien Di Mascio's avatar
Adrien Di Mascio committed
419
def update_cubes_catalogs(cubes):
420
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
421
    for cubedir in cubes:
422
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
423
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
424
            continue
425
        try:
426
            toedit = update_cube_catalogs(cubedir)
427
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
428
429
430
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
431
432
433
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
434
            print('*** error while updating catalogs for cube', cubedir)
435
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
436
437
        else:
            # instructions pour la suite
438
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
439
440
441
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
442
443
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
444
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
445

446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498

class I18nCubeMessageExtractor(object):
    """This class encapsulates all the xgettext extraction logic

    ``generate_pot_file`` is the main entry point called by the ``i18ncube``
    command. A cube might decide to customize extractors to ignore a given
    directory or to extract messages from a new file type (e.g. .jinja2 files)

    For each file type, the class must define two methods:

    - ``collect_{filetype}()`` that must return the list of files
      xgettext should inspect,

    - ``extract_{filetype}(files)`` that calls xgettext and returns the
      path to the generated ``pot`` file
    """
    blacklist = STD_BLACKLIST
    formats = ['tal', 'js', 'py']

    def __init__(self, workdir, cubedir):
        self.workdir = workdir
        self.cubedir = cubedir

    def generate_pot_file(self):
        """main entry point: return the generated ``cube.pot`` file

        This function first generates all the pot files (schema, tal,
        py, js) and then merges them in a single ``cube.pot`` that will
        be used to eventually update the ``i18n/*.po`` files.
        """
        potfiles = self.generate_pot_files()
        potfile = osp.join(self.workdir, 'cube.pot')
        print('-> merging %i .pot files' % len(potfiles))
        cmd = ['msgcat', '-o', potfile]
        cmd.extend(potfiles)
        execute2(cmd)
        return potfile if osp.exists(potfile) else None

    def find(self, exts, blacklist=None):
        """collect files with extensions ``exts`` in the cube directory
        """
        if blacklist is None:
            blacklist = self.blacklist
        return find(self.cubedir, exts, blacklist=blacklist)

    def generate_pot_files(self):
        """generate and return the list of all ``pot`` files for the cube

        - static-messages.pot,
        - schema.pot,
        - one ``pot`` file for each inspected format (.py, .js, etc.)
        """
        print('-> extracting messages:', end=' ')
499
        potfiles = []
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
        # static messages
        if osp.exists(osp.join('i18n', 'entities.pot')):
            warn('entities.pot is deprecated, rename file '
                 'to static-messages.pot (%s)'
                 % osp.join('i18n', 'entities.pot'), DeprecationWarning)
            potfiles.append(osp.join('i18n', 'entities.pot'))
        elif osp.exists(osp.join('i18n', 'static-messages.pot')):
            potfiles.append(osp.join('i18n', 'static-messages.pot'))
        # messages from schema
        potfiles.append(self.schemapot())
        # messages from sourcecode
        for fmt in self.formats:
            collector = getattr(self, 'collect_{0}'.format(fmt))
            extractor = getattr(self, 'extract_{0}'.format(fmt))
            files = collector()
            if files:
                potfile = extractor(files)
                if potfile:
                    potfiles.append(potfile)
        return potfiles

    def schemapot(self):
        """generate the ``schema.pot`` file"""
        schemapot = osp.join(self.workdir, 'schema.pot')
        print('schema', end=' ')
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
        schemapotstream = open(schemapot, 'w')
        generate_schema_pot(schemapotstream.write, self.cubedir)
        schemapotstream.close()
        return schemapot

    def _xgettext(self, files, output, k='_', extraopts=''):
        """shortcut to execute the xgettext command and return output file
        """
        tmppotfile = osp.join(self.workdir, output)
        cmd = ['xgettext', '--no-location', '--omit-header', '-k' + k,
               '-o', tmppotfile] + extraopts.split() + files
538
        execute2(cmd)
539
        if osp.exists(tmppotfile):
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
            return tmppotfile

    def collect_tal(self):
        print('TAL', end=' ')
        return self.find(('.py', '.pt'))

    def extract_tal(self, files):
        tali18nfile = osp.join(self.workdir, 'tali18n.py')
        extract_from_tal(files, tali18nfile)
        return self._xgettext(files, output='tal.pot')

    def collect_js(self):
        print('Javascript')
        return [jsfile for jsfile in self.find('.js')
                if osp.basename(jsfile).startswith('cub')]

    def extract_js(self, files):
        return self._xgettext(files, output='js.pot',
                              extraopts='-L java --from-code=utf-8')

    def collect_py(self):
        print('-> creating cube-specific catalog')
        return self.find('.py')

    def extract_py(self, files):
        return self._xgettext(files, output='py.pot')


def update_cube_catalogs(cubedir):
    cubedir = osp.abspath(osp.normpath(cubedir))
    workdir = tempfile.mkdtemp()
571
    try:
572
573
574
575
576
577
        cubename = osp.basename(cubedir)
        if cubename.startswith('cubicweb_'):  # new layout
            distname = cubename
            cubename = cubename[len('cubicweb_'):]
        else:
            distname = 'cubicweb_' + cubename
578
        print('cubedir', cubedir)
579
580
581
582
583
584
585
        extract_cls = I18nCubeMessageExtractor
        try:
            extract_cls = pkg_resources.load_entry_point(
                distname, 'cubicweb.i18ncube', cubename)
        except (pkg_resources.DistributionNotFound, ImportError):
            pass  # no customization found
        print(underline_title('Updating i18n catalogs for cube %s' % cubename))
586
        chdir(cubedir)
587
        extractor = extract_cls(workdir, cubedir)
588
589
        potfile = extractor.generate_pot_file()
        if potfile is None:
590
            print('no message catalog for cube', cubename, 'nothing to translate')
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
            return ()
        print('-> merging main pot file with existing translations:', end=' ')
        chdir('i18n')
        toedit = []
        for lang in CubicWebNoAppConfiguration.cw_languages():
            print(lang, end=' ')
            cubepo = '%s.po' % lang
            if not osp.exists(cubepo):
                shutil.copy(potfile, cubepo)
            else:
                cmd = ['msgmerge', '-N', '-s', '-o', cubepo + 'new',
                       cubepo, potfile]
                execute2(cmd)
                ensure_fs_mode(cubepo)
                shutil.move('%snew' % cubepo, cubepo)
            toedit.append(osp.abspath(cubepo))
        print()
        return toedit
    finally:
        # cleanup
611
        shutil.rmtree(workdir)
612
613


614
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
615
616
617
    """Create a new cube.

    <cubename>
618
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
619
620
621
    """
    name = 'newcube'
    arguments = '<cubename>'
622
    min_args = max_args = 1
623
    options = (
624
625
626
627
628
629
630
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
631
632
633
634
635
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
636
637
638
639
640
641
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
660
661
662
663
664
665
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
666
667
        )

668
    LICENSES = {
669
670
671
672
673
674
675
676
677
678
679
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
680
681
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
682
''',
683

684
685
686
687
688
689
690
691
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
692
693
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
694
695
696
697
698
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
699
        }
700

Adrien Di Mascio's avatar
Adrien Di Mascio committed
701
    def run(self, args):
702
        import re
703
        from logilab.common.shellutils import ASK
704
705
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
706
707
            raise BadCommandUsage(
                'cube name must be a valid python module name')
708
        verbose = self.get('verbose')
709
710
        destdir = self.get('directory')
        if not destdir:
711
            cubespath = ServerConfiguration.cubes_search_path()
712
            if len(cubespath) > 1:
713
714
715
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
716
717
718
            destdir = cubespath[0]
        if not osp.isdir(destdir):
            print("-> creating cubes directory", destdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
719
            try:
720
                mkdir(destdir)
721
            except OSError as err:
722
                self.fail("failed to create directory %r\n(%s)"
723
                          % (destdir, err))
724
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
725
        if verbose:
726
            distname = input('Debian name for your cube ? [%s]): '
727
                             % default_name).strip()
728
            if not distname:
729
                distname = default_name
730
            elif not distname.startswith('cubicweb-'):
731
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
732
733
                    distname = 'cubicweb-' + distname
        else:
734
            distname = default_name
735
        if not re.match('[a-z][-a-z0-9]*$', distname):
736
737
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
738
739
740
741
        cubedir = osp.join(destdir, distname)
        if osp.exists(cubedir):
            self.fail("%s already exists!" % cubedir)
        skeldir = osp.join(BASEDIR, 'skeleton')
742
        longdesc = shortdesc = input(
743
            'Enter a short description for your cube: ')
744
        if verbose:
745
            longdesc = input(
746
                'Enter a long description (leave empty to reuse the short one): ')
747
748
749
750
        dependencies = {
            'six': '>= 1.4.0',
            'cubicweb': '>= %s' % cubicwebversion,
        }
751
        if verbose:
752
            dependencies.update(self._ask_for_dependencies())
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
        context = {
            'cubename': cubename,
            'distname': distname,
            'shortdesc': shortdesc,
            'longdesc': longdesc or shortdesc,
            'dependencies': dependencies,
            'version': cubicwebversion,
            'year': str(date.today().year),
            'author': self['author'],
            'author-email': self['author-email'],
            'rfc2822-date': datetime.now(tz=UTC).strftime('%a, %d %b %Y %T %z'),
            'author-web-site': self['author-web-site'],
            'license': self['license'],
            'long-license': self.LICENSES[self['license']],
        }
768
769
770
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
771
                        'cubes.*', 'uiprops.py*')
772
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
773

774
    def _ask_for_dependencies(self):
775
776
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
777
778
779
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
780
781
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
782
                depcubes.append(cube)
783
            if answer == 'type':
784
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
785
                break
786
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
787
                break
788
789
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
790

791
792
793
794

class ExamineLogCommand(Command):
    """Examine a rql log file.

795
    Will print out the following table
796

797
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
798

799
    sorted by descending cumulative time (clock). Time are expressed in seconds.
800

801
    Chances are the lines at the top are the ones that will bring the higher
802
    benefit after optimisation. Start there.
803
    """
804
    arguments = 'rql.log'
805
    name = 'exlog'
806
    options = ()
807

808
809
810
    def run(self, args):
        import re
        requests = {}
811
        for filepath in args:
812
            try:
Rémi Cardona's avatar
Rémi Cardona committed
813
                stream = open(filepath)
814
            except OSError as ex:
815
816
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
817
            for lineno, line in enumerate(stream):
818
819
820
821
822
823
824
825
826
827
828
829
830
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
831
                except Exception as exc:
832
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
833
        stat = []
834
        for rql, times in requests.items():
835
836
837
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
838
839
        stat.sort()
        stat.reverse()
840
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
841
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
842
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
843
844
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
845

846

847
848
849
850
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
851
    min_args = max_args = 1
852
853
    options = [
        ('output-file',
854
         {'type':'string', 'default': None,
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
888
889
890
891
892
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
893
        ]
894
895

    def run(self, args):
896
897
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
898
        from logilab.common.textutils import splitstrip
899
900
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
901
902
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
903
        cubes = splitstrip(args[0])
904
        dev_conf = DevConfiguration(*cubes)
905
906
907
908
909
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
910
911
912
913
914
915
916
917
918
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

935
936
937
938
        if viewer:
            p = Popen((viewer, out))
            p.wait()

939

940
for cmdcls in (UpdateCubicWebCatalogCommand,
941
               UpdateCubeCatalogCommand,
942
943
944
945
946
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)