devctl.py 36.2 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

23
24
25
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
26
27
28

import shutil
import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
29
import sys
30
from datetime import datetime, date
31
from os import mkdir, chdir, path as osp
32
import pkg_resources
33
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
34

35
from pytz import UTC
36

37
38
from six.moves import input

Adrien Di Mascio's avatar
Adrien Di Mascio committed
39
from logilab.common import STD_BLACKLIST
40
from logilab.common.modutils import clean_sys_modules
41
42
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find
43

Adrien Di Mascio's avatar
Adrien Di Mascio committed
44
from cubicweb.__pkginfo__ import version as cubicwebversion
45
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
46
from cubicweb.i18n import extract_from_tal, execute2
47
from cubicweb.cwctl import CWCTL
48
from cubicweb.cwconfig import CubicWebNoAppConfiguration
49
50
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
51
52
53
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

54

55
56
57
__docformat__ = "restructuredtext en"


58
59
60
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')
61
STD_BLACKLIST.add('node_modules')
62
63


64
65
66
67
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
68
    creating = True
69
    cleanup_unused_appobjects = False
70
71
72
73
74

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
75

76
    def __init__(self, *cubes):
77
78
79
80
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
81
            self.load_site_cubicweb()
82
83
        else:
            self._cubes = ()
84

Adrien Di Mascio's avatar
Adrien Di Mascio committed
85
86
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
87
        return None
88
89
90
91

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
92
93
    def main_config_file(self):
        return None
94
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
95
        pass
96
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
97
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
98
99
    def default_log_file(self):
        return None
100
101
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
102
103


Adrien Di Mascio's avatar
Adrien Di Mascio committed
104
105
106
107
108
109
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
110
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
111
    if cubedir:
112
        cube = osp.split(cubedir)[-1]
113
114
        if cube.startswith('cubicweb_'):
            cube = cube[len('cubicweb_'):]
115
116
117
118
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
119
    else:
120
121
        config = DevConfiguration()
        cube = libconfig = None
122
    clean_sys_modules(config.appobjects_modnames())
123
    schema = config.load_schema(remove_unused_rtypes=False)
124
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
125
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
126
127
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
128
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
129
130


131
def _generate_schema_pot(w, vreg, schema, libconfig=None):
132
    from cubicweb.i18n import add_msg
133
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
134
    w('# schema pot file, generated on %s\n'
135
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
136
137
138
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
139
    vregdone = set()
140
    afss = vreg['uicfg']['autoform_section']
141
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
142
    if libconfig is not None:
143
144
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
145
        from cubicweb.cwvreg import CWRegistryStore
146
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
147
        clean_sys_modules(libconfig.appobjects_modnames())
148
        libvreg = CWRegistryStore(libconfig)
149
        libvreg.set_schema(libschema) # trigger objects registration
150
        libafss = libvreg['uicfg']['autoform_section']
151
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
152
153
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
154
155
156
157

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
158
    else:
159
        # processing cubicweb itself
160
        libschema = {}
161
162
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
163
164
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
165
    done = set()
166
167
168
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
169
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
170
        etype = eschema.type
171
172
173
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
174
            if not eschema.final:
175
                add_msg(w, 'This %s:' % etype)
176
                add_msg(w, 'New %s' % etype)
177
                add_msg(w, 'add a %s' % etype) # AddNewAction
178
179
180
181
182
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
183
184
185
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
186
        if eschema.final:
187
188
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
189
190
            if rschema.final:
                continue
191
            for tschema in targetschemas:
192

193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
220
221
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
222
    w('# subject and object forms for each relation type\n')
223
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
224
    w('\n')
225
226
227
228
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
229
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
230
        rtype = rschema.type
231
        if rtype not in libschema:
232
233
            # bw compat, necessary until all translation of relation are done
            # properly...
234
            add_msg(w, rtype)
235
            done.add(rtype)
236
237
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
238
            done.add(rschema.description)
239
240
241
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
242
        # add context information only for non-metadata rtypes
243
        if rschema not in NO_I18NCONTEXT:
244
            libsubjects = librschema and librschema.subjects() or ()
245
            for subjschema in rschema.subjects():
246
247
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
248
        if not (rschema.final or rschema.symmetric):
249
            if rschema not in NO_I18NCONTEXT:
250
251
252
253
254
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
255
256
                # bw compat, necessary until all translation of relation are
                # done properly...
257
                add_msg(w, '%s_object' % rtype)
258
        for rdef in rschema.rdefs.values():
259
260
261
262
263
264
265
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
266
    for objid in _iter_vreg_objids(vreg, vregdone):
267
268
269
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
270

271
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
272
    for reg, objdict in vreg.items():
273
274
        if reg in ('boxes', 'contentnavigation'):
            continue
275
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
276
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
277
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
278
                if objid in done:
279
                    break
280
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
281
                if pdefs:
282
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
283
                    done.add(objid)
284
                    break
285

286

Adrien Di Mascio's avatar
Adrien Di Mascio committed
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
304

Adrien Di Mascio's avatar
Adrien Di Mascio committed
305
306
307
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
308
    name = 'i18ncubicweb'
309
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
310
311
312
313

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
314
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
315
316
        import yams
        from logilab.common.fileutils import ensure_fs_mode
317
        from logilab.common.shellutils import globfind, find, rm
318
        from logilab.common.modutils import get_module_files
319
        from cubicweb.i18n import extract_from_tal, execute2
320
        tempdir = tempfile.mkdtemp(prefix='cw-')
321
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
322
323
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
324
325
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
326
327
328
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
329
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
330
331
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
332
        print('TAL', end=' ')
333
334
335
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
336
        print('-> generate .pot files.')
337
338
339
340
341
342
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
343
344
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
345
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
346
                                ]:
347
            potfile = osp.join(tempdir, '%s.pot' % id)
348
349
350
351
352
353
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
354
            if osp.exists(potfile):
355
356
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
357
358
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
359
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
360
361
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
362
        print('-> merging main pot file with existing translations.')
363
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
364
        toedit = []
365
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
366
            target = '%s.po' % lang
367
368
369
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
370
371
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
372
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
373
        # cleanup
374
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
375
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
376
377
378
379
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
380
381


382
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
383
384
385
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
386
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
387
    arguments = '[<cube>...]'
388

Adrien Di Mascio's avatar
Adrien Di Mascio committed
389
390
391
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
392
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
393
        else:
394
395
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
396
397
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
398
399
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
400

401

Adrien Di Mascio's avatar
Adrien Di Mascio committed
402
def update_cubes_catalogs(cubes):
403
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
404
    for cubedir in cubes:
405
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
406
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
407
            continue
408
        try:
409
            toedit = update_cube_catalogs(cubedir)
410
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
411
412
413
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
414
415
416
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
417
            print('*** error while updating catalogs for cube', cubedir)
418
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
419
420
        else:
            # instructions pour la suite
421
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
422
423
424
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
425
426
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
427
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
428

429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481

class I18nCubeMessageExtractor(object):
    """This class encapsulates all the xgettext extraction logic

    ``generate_pot_file`` is the main entry point called by the ``i18ncube``
    command. A cube might decide to customize extractors to ignore a given
    directory or to extract messages from a new file type (e.g. .jinja2 files)

    For each file type, the class must define two methods:

    - ``collect_{filetype}()`` that must return the list of files
      xgettext should inspect,

    - ``extract_{filetype}(files)`` that calls xgettext and returns the
      path to the generated ``pot`` file
    """
    blacklist = STD_BLACKLIST
    formats = ['tal', 'js', 'py']

    def __init__(self, workdir, cubedir):
        self.workdir = workdir
        self.cubedir = cubedir

    def generate_pot_file(self):
        """main entry point: return the generated ``cube.pot`` file

        This function first generates all the pot files (schema, tal,
        py, js) and then merges them in a single ``cube.pot`` that will
        be used to eventually update the ``i18n/*.po`` files.
        """
        potfiles = self.generate_pot_files()
        potfile = osp.join(self.workdir, 'cube.pot')
        print('-> merging %i .pot files' % len(potfiles))
        cmd = ['msgcat', '-o', potfile]
        cmd.extend(potfiles)
        execute2(cmd)
        return potfile if osp.exists(potfile) else None

    def find(self, exts, blacklist=None):
        """collect files with extensions ``exts`` in the cube directory
        """
        if blacklist is None:
            blacklist = self.blacklist
        return find(self.cubedir, exts, blacklist=blacklist)

    def generate_pot_files(self):
        """generate and return the list of all ``pot`` files for the cube

        - static-messages.pot,
        - schema.pot,
        - one ``pot`` file for each inspected format (.py, .js, etc.)
        """
        print('-> extracting messages:', end=' ')
482
        potfiles = []
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
        # static messages
        if osp.exists(osp.join('i18n', 'entities.pot')):
            warn('entities.pot is deprecated, rename file '
                 'to static-messages.pot (%s)'
                 % osp.join('i18n', 'entities.pot'), DeprecationWarning)
            potfiles.append(osp.join('i18n', 'entities.pot'))
        elif osp.exists(osp.join('i18n', 'static-messages.pot')):
            potfiles.append(osp.join('i18n', 'static-messages.pot'))
        # messages from schema
        potfiles.append(self.schemapot())
        # messages from sourcecode
        for fmt in self.formats:
            collector = getattr(self, 'collect_{0}'.format(fmt))
            extractor = getattr(self, 'extract_{0}'.format(fmt))
            files = collector()
            if files:
                potfile = extractor(files)
                if potfile:
                    potfiles.append(potfile)
        return potfiles

    def schemapot(self):
        """generate the ``schema.pot`` file"""
        schemapot = osp.join(self.workdir, 'schema.pot')
        print('schema', end=' ')
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
        schemapotstream = open(schemapot, 'w')
        generate_schema_pot(schemapotstream.write, self.cubedir)
        schemapotstream.close()
        return schemapot

    def _xgettext(self, files, output, k='_', extraopts=''):
        """shortcut to execute the xgettext command and return output file
        """
        tmppotfile = osp.join(self.workdir, output)
        cmd = ['xgettext', '--no-location', '--omit-header', '-k' + k,
               '-o', tmppotfile] + extraopts.split() + files
521
        execute2(cmd)
522
        if osp.exists(tmppotfile):
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
            return tmppotfile

    def collect_tal(self):
        print('TAL', end=' ')
        return self.find(('.py', '.pt'))

    def extract_tal(self, files):
        tali18nfile = osp.join(self.workdir, 'tali18n.py')
        extract_from_tal(files, tali18nfile)
        return self._xgettext(files, output='tal.pot')

    def collect_js(self):
        print('Javascript')
        return [jsfile for jsfile in self.find('.js')
                if osp.basename(jsfile).startswith('cub')]

    def extract_js(self, files):
        return self._xgettext(files, output='js.pot',
                              extraopts='-L java --from-code=utf-8')

    def collect_py(self):
        print('-> creating cube-specific catalog')
        return self.find('.py')

    def extract_py(self, files):
        return self._xgettext(files, output='py.pot')


def update_cube_catalogs(cubedir):
    cubedir = osp.abspath(osp.normpath(cubedir))
    workdir = tempfile.mkdtemp()
554
    try:
555
556
557
558
559
560
        cubename = osp.basename(cubedir)
        if cubename.startswith('cubicweb_'):  # new layout
            distname = cubename
            cubename = cubename[len('cubicweb_'):]
        else:
            distname = 'cubicweb_' + cubename
561
        print('cubedir', cubedir)
562
563
564
565
566
567
568
        extract_cls = I18nCubeMessageExtractor
        try:
            extract_cls = pkg_resources.load_entry_point(
                distname, 'cubicweb.i18ncube', cubename)
        except (pkg_resources.DistributionNotFound, ImportError):
            pass  # no customization found
        print(underline_title('Updating i18n catalogs for cube %s' % cubename))
569
        chdir(cubedir)
570
        extractor = extract_cls(workdir, cubedir)
571
572
        potfile = extractor.generate_pot_file()
        if potfile is None:
573
            print('no message catalog for cube', cubename, 'nothing to translate')
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
            return ()
        print('-> merging main pot file with existing translations:', end=' ')
        chdir('i18n')
        toedit = []
        for lang in CubicWebNoAppConfiguration.cw_languages():
            print(lang, end=' ')
            cubepo = '%s.po' % lang
            if not osp.exists(cubepo):
                shutil.copy(potfile, cubepo)
            else:
                cmd = ['msgmerge', '-N', '-s', '-o', cubepo + 'new',
                       cubepo, potfile]
                execute2(cmd)
                ensure_fs_mode(cubepo)
                shutil.move('%snew' % cubepo, cubepo)
            toedit.append(osp.abspath(cubepo))
        print()
        return toedit
    finally:
        # cleanup
594
        shutil.rmtree(workdir)
595
596


597
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
598
599
600
    """Create a new cube.

    <cubename>
601
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
602
603
604
    """
    name = 'newcube'
    arguments = '<cubename>'
605
    min_args = max_args = 1
606
    options = (
607
608
609
610
611
612
613
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
614
615
616
617
618
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
619
620
621
622
623
624
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
643
644
645
646
647
648
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
649
650
        )

651
    LICENSES = {
652
653
654
655
656
657
658
659
660
661
662
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
663
664
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
665
''',
666

667
668
669
670
671
672
673
674
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
675
676
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
677
678
679
680
681
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
682
        }
683

Adrien Di Mascio's avatar
Adrien Di Mascio committed
684
    def run(self, args):
685
        import re
686
        from logilab.common.shellutils import ASK
687
688
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
689
690
            raise BadCommandUsage(
                'cube name must be a valid python module name')
691
        verbose = self.get('verbose')
692
693
        destdir = self.get('directory')
        if not destdir:
694
            cubespath = ServerConfiguration.cubes_search_path()
695
            if len(cubespath) > 1:
696
697
698
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
699
700
701
            destdir = cubespath[0]
        if not osp.isdir(destdir):
            print("-> creating cubes directory", destdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
702
            try:
703
                mkdir(destdir)
704
            except OSError as err:
705
                self.fail("failed to create directory %r\n(%s)"
706
                          % (destdir, err))
707
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
708
        if verbose:
709
            distname = input('Debian name for your cube ? [%s]): '
710
                             % default_name).strip()
711
            if not distname:
712
                distname = default_name
713
            elif not distname.startswith('cubicweb-'):
714
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
715
716
                    distname = 'cubicweb-' + distname
        else:
717
            distname = default_name
718
        if not re.match('[a-z][-a-z0-9]*$', distname):
719
720
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
721
722
723
724
        cubedir = osp.join(destdir, distname)
        if osp.exists(cubedir):
            self.fail("%s already exists!" % cubedir)
        skeldir = osp.join(BASEDIR, 'skeleton')
725
        longdesc = shortdesc = input(
726
            'Enter a short description for your cube: ')
727
        if verbose:
728
            longdesc = input(
729
                'Enter a long description (leave empty to reuse the short one): ')
730
731
732
733
        dependencies = {
            'six': '>= 1.4.0',
            'cubicweb': '>= %s' % cubicwebversion,
        }
734
        if verbose:
735
            dependencies.update(self._ask_for_dependencies())
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
        context = {
            'cubename': cubename,
            'distname': distname,
            'shortdesc': shortdesc,
            'longdesc': longdesc or shortdesc,
            'dependencies': dependencies,
            'version': cubicwebversion,
            'year': str(date.today().year),
            'author': self['author'],
            'author-email': self['author-email'],
            'rfc2822-date': datetime.now(tz=UTC).strftime('%a, %d %b %Y %T %z'),
            'author-web-site': self['author-web-site'],
            'license': self['license'],
            'long-license': self.LICENSES[self['license']],
        }
751
752
753
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
754
                        'cubes.*', 'uiprops.py*')
755
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
756

757
    def _ask_for_dependencies(self):
758
759
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
760
761
762
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
763
764
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
765
                depcubes.append(cube)
766
            if answer == 'type':
767
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
768
                break
769
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
770
                break
771
772
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
773

774
775
776
777

class ExamineLogCommand(Command):
    """Examine a rql log file.

778
    Will print out the following table
779

780
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
781

782
    sorted by descending cumulative time (clock). Time are expressed in seconds.
783

784
    Chances are the lines at the top are the ones that will bring the higher
785
    benefit after optimisation. Start there.
786
    """
787
    arguments = 'rql.log'
788
    name = 'exlog'
789
    options = ()
790

791
792
793
    def run(self, args):
        import re
        requests = {}
794
        for filepath in args:
795
            try:
Rémi Cardona's avatar
Rémi Cardona committed
796
                stream = open(filepath)
797
            except OSError as ex:
798
799
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
800
            for lineno, line in enumerate(stream):
801
802
803
804
805
806
807
808
809
810
811
812
813
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
814
                except Exception as exc:
815
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
816
        stat = []
817
        for rql, times in requests.items():
818
819
820
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
821
822
        stat.sort()
        stat.reverse()
823
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
824
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
825
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
826
827
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
828

829

830
831
832
833
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
834
    min_args = max_args = 1
835
836
    options = [
        ('output-file',
837
         {'type':'string', 'default': None,
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
871
872
873
874
875
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
876
        ]
877
878

    def run(self, args):
879
880
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
881
        from logilab.common.textutils import splitstrip
882
883
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
884
885
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
886
        cubes = splitstrip(args[0])
887
        dev_conf = DevConfiguration(*cubes)
888
889
890
891
892
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
893
894
895
896
897
898
899
900
901
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

918
919
920
921
        if viewer:
            p = Popen((viewer, out))
            p.wait()

922

923
for cmdcls in (UpdateCubicWebCatalogCommand,
924
               UpdateCubeCatalogCommand,
925
926
927
928
929
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)