devctl.py 36.6 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

23
24
25
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
26
27
28

import shutil
import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
29
import sys
30
from datetime import datetime, date
31
from os import mkdir, chdir, path as osp
32
import pkg_resources
33
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
34

35
from pytz import UTC
36

37
38
from six.moves import input

Adrien Di Mascio's avatar
Adrien Di Mascio committed
39
from logilab.common import STD_BLACKLIST
40
41
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find
42

Adrien Di Mascio's avatar
Adrien Di Mascio committed
43
from cubicweb.__pkginfo__ import version as cubicwebversion
44
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
45
from cubicweb.i18n import extract_from_tal, execute2
46
from cubicweb.cwctl import CWCTL
47
from cubicweb.cwconfig import CubicWebNoAppConfiguration
48
49
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
50
51
52
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

53

54
55
56
__docformat__ = "restructuredtext en"


57
58
59
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')
60
STD_BLACKLIST.add('node_modules')
61
62


63
64
65
66
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
67
    creating = True
68
    cleanup_unused_appobjects = False
69
70
71
72
73

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
74

75
    def __init__(self, *cubes):
76
77
78
79
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
80
            self.load_site_cubicweb()
81
82
        else:
            self._cubes = ()
83

Adrien Di Mascio's avatar
Adrien Di Mascio committed
84
85
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
86
        return None
87
88
89
90

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
91
92
    def main_config_file(self):
        return None
93
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
94
        pass
95
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
96
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
97
98
    def default_log_file(self):
        return None
99
100
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
101
102
103
104


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
105
    appobjects_path = config.appobjects_path()
106
    for name, mod in list(sys.modules.items()):
Sylvain Thenault's avatar
Sylvain Thenault committed
107
108
109
110
111
112
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
113
114
115
        if mod.__file__ is None:
            # odd/rare but real
            continue
116
        for path in appobjects_path:
Sylvain Thenault's avatar
Sylvain Thenault committed
117
118
119
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
120

Adrien Di Mascio's avatar
Adrien Di Mascio committed
121
122
123
124
125
126
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
127
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
128
    if cubedir:
129
        cube = osp.split(cubedir)[-1]
130
131
        if cube.startswith('cubicweb_'):
            cube = cube[len('cubicweb_'):]
132
133
134
135
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
136
    else:
137
138
139
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
140
    schema = config.load_schema(remove_unused_rtypes=False)
141
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
142
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
143
144
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
145
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
146
147


148
def _generate_schema_pot(w, vreg, schema, libconfig=None):
149
    from cubicweb.i18n import add_msg
150
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
151
    w('# schema pot file, generated on %s\n'
152
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
153
154
155
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
156
    vregdone = set()
157
    afss = vreg['uicfg']['autoform_section']
158
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
159
    if libconfig is not None:
160
161
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
162
        from cubicweb.cwvreg import CWRegistryStore
163
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
164
        cleanup_sys_modules(libconfig)
165
        libvreg = CWRegistryStore(libconfig)
166
        libvreg.set_schema(libschema) # trigger objects registration
167
        libafss = libvreg['uicfg']['autoform_section']
168
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
169
170
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
171
172
173
174

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
175
    else:
176
        # processing cubicweb itself
177
        libschema = {}
178
179
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
180
181
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
182
    done = set()
183
184
185
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
186
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
187
        etype = eschema.type
188
189
190
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
191
            if not eschema.final:
192
                add_msg(w, 'This %s:' % etype)
193
                add_msg(w, 'New %s' % etype)
194
                add_msg(w, 'add a %s' % etype) # AddNewAction
195
196
197
198
199
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
200
201
202
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
203
        if eschema.final:
204
205
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
206
207
            if rschema.final:
                continue
208
            for tschema in targetschemas:
209

210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
237
238
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
239
    w('# subject and object forms for each relation type\n')
240
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
241
    w('\n')
242
243
244
245
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
246
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
247
        rtype = rschema.type
248
        if rtype not in libschema:
249
250
            # bw compat, necessary until all translation of relation are done
            # properly...
251
            add_msg(w, rtype)
252
            done.add(rtype)
253
254
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
255
            done.add(rschema.description)
256
257
258
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
259
        # add context information only for non-metadata rtypes
260
        if rschema not in NO_I18NCONTEXT:
261
            libsubjects = librschema and librschema.subjects() or ()
262
            for subjschema in rschema.subjects():
263
264
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
265
        if not (rschema.final or rschema.symmetric):
266
            if rschema not in NO_I18NCONTEXT:
267
268
269
270
271
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
272
273
                # bw compat, necessary until all translation of relation are
                # done properly...
274
                add_msg(w, '%s_object' % rtype)
275
        for rdef in rschema.rdefs.values():
276
277
278
279
280
281
282
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
283
    for objid in _iter_vreg_objids(vreg, vregdone):
284
285
286
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
287

288
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
289
    for reg, objdict in vreg.items():
290
291
        if reg in ('boxes', 'contentnavigation'):
            continue
292
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
293
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
294
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
295
                if objid in done:
296
                    break
297
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
298
                if pdefs:
299
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
300
                    done.add(objid)
301
                    break
302

303

Adrien Di Mascio's avatar
Adrien Di Mascio committed
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
321

Adrien Di Mascio's avatar
Adrien Di Mascio committed
322
323
324
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
325
    name = 'i18ncubicweb'
326
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
327
328
329
330

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
331
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
332
333
        import yams
        from logilab.common.fileutils import ensure_fs_mode
334
        from logilab.common.shellutils import globfind, find, rm
335
        from logilab.common.modutils import get_module_files
336
        from cubicweb.i18n import extract_from_tal, execute2
337
        tempdir = tempfile.mkdtemp(prefix='cw-')
338
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
339
340
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
341
342
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
343
344
345
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
346
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
347
348
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
349
        print('TAL', end=' ')
350
351
352
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
353
        print('-> generate .pot files.')
354
355
356
357
358
359
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
360
361
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
362
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
363
                                ]:
364
            potfile = osp.join(tempdir, '%s.pot' % id)
365
366
367
368
369
370
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
371
            if osp.exists(potfile):
372
373
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
374
375
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
376
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
377
378
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
379
        print('-> merging main pot file with existing translations.')
380
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
381
        toedit = []
382
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
383
            target = '%s.po' % lang
384
385
386
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
387
388
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
389
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
390
        # cleanup
391
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
392
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
393
394
395
396
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
397
398


399
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
400
401
402
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
403
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
404
    arguments = '[<cube>...]'
405

Adrien Di Mascio's avatar
Adrien Di Mascio committed
406
407
408
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
409
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
410
        else:
411
412
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
413
414
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
415
416
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
417

418

Adrien Di Mascio's avatar
Adrien Di Mascio committed
419
def update_cubes_catalogs(cubes):
420
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
421
    for cubedir in cubes:
422
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
423
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
424
            continue
425
        try:
426
            toedit = update_cube_catalogs(cubedir)
427
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
428
429
430
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
431
432
433
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
434
            print('*** error while updating catalogs for cube', cubedir)
435
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
436
437
        else:
            # instructions pour la suite
438
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
439
440
441
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
442
443
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
444
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
445

446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498

class I18nCubeMessageExtractor(object):
    """This class encapsulates all the xgettext extraction logic

    ``generate_pot_file`` is the main entry point called by the ``i18ncube``
    command. A cube might decide to customize extractors to ignore a given
    directory or to extract messages from a new file type (e.g. .jinja2 files)

    For each file type, the class must define two methods:

    - ``collect_{filetype}()`` that must return the list of files
      xgettext should inspect,

    - ``extract_{filetype}(files)`` that calls xgettext and returns the
      path to the generated ``pot`` file
    """
    blacklist = STD_BLACKLIST
    formats = ['tal', 'js', 'py']

    def __init__(self, workdir, cubedir):
        self.workdir = workdir
        self.cubedir = cubedir

    def generate_pot_file(self):
        """main entry point: return the generated ``cube.pot`` file

        This function first generates all the pot files (schema, tal,
        py, js) and then merges them in a single ``cube.pot`` that will
        be used to eventually update the ``i18n/*.po`` files.
        """
        potfiles = self.generate_pot_files()
        potfile = osp.join(self.workdir, 'cube.pot')
        print('-> merging %i .pot files' % len(potfiles))
        cmd = ['msgcat', '-o', potfile]
        cmd.extend(potfiles)
        execute2(cmd)
        return potfile if osp.exists(potfile) else None

    def find(self, exts, blacklist=None):
        """collect files with extensions ``exts`` in the cube directory
        """
        if blacklist is None:
            blacklist = self.blacklist
        return find(self.cubedir, exts, blacklist=blacklist)

    def generate_pot_files(self):
        """generate and return the list of all ``pot`` files for the cube

        - static-messages.pot,
        - schema.pot,
        - one ``pot`` file for each inspected format (.py, .js, etc.)
        """
        print('-> extracting messages:', end=' ')
499
        potfiles = []
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
        # static messages
        if osp.exists(osp.join('i18n', 'entities.pot')):
            warn('entities.pot is deprecated, rename file '
                 'to static-messages.pot (%s)'
                 % osp.join('i18n', 'entities.pot'), DeprecationWarning)
            potfiles.append(osp.join('i18n', 'entities.pot'))
        elif osp.exists(osp.join('i18n', 'static-messages.pot')):
            potfiles.append(osp.join('i18n', 'static-messages.pot'))
        # messages from schema
        potfiles.append(self.schemapot())
        # messages from sourcecode
        for fmt in self.formats:
            collector = getattr(self, 'collect_{0}'.format(fmt))
            extractor = getattr(self, 'extract_{0}'.format(fmt))
            files = collector()
            if files:
                potfile = extractor(files)
                if potfile:
                    potfiles.append(potfile)
        return potfiles

    def schemapot(self):
        """generate the ``schema.pot`` file"""
        schemapot = osp.join(self.workdir, 'schema.pot')
        print('schema', end=' ')
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
        schemapotstream = open(schemapot, 'w')
        generate_schema_pot(schemapotstream.write, self.cubedir)
        schemapotstream.close()
        return schemapot

    def _xgettext(self, files, output, k='_', extraopts=''):
        """shortcut to execute the xgettext command and return output file
        """
        tmppotfile = osp.join(self.workdir, output)
        cmd = ['xgettext', '--no-location', '--omit-header', '-k' + k,
               '-o', tmppotfile] + extraopts.split() + files
538
        execute2(cmd)
539
        if osp.exists(tmppotfile):
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
            return tmppotfile

    def collect_tal(self):
        print('TAL', end=' ')
        return self.find(('.py', '.pt'))

    def extract_tal(self, files):
        tali18nfile = osp.join(self.workdir, 'tali18n.py')
        extract_from_tal(files, tali18nfile)
        return self._xgettext(files, output='tal.pot')

    def collect_js(self):
        print('Javascript')
        return [jsfile for jsfile in self.find('.js')
                if osp.basename(jsfile).startswith('cub')]

    def extract_js(self, files):
        return self._xgettext(files, output='js.pot',
                              extraopts='-L java --from-code=utf-8')

    def collect_py(self):
        print('-> creating cube-specific catalog')
        return self.find('.py')

    def extract_py(self, files):
        return self._xgettext(files, output='py.pot')


def update_cube_catalogs(cubedir):
    cubedir = osp.abspath(osp.normpath(cubedir))
    workdir = tempfile.mkdtemp()
571
    try:
572
573
        distname = osp.basename(cubedir)
        cubename = distname.split('_')[-1]
574
        print('cubedir', cubedir)
575
576
577
578
579
580
581
        extract_cls = I18nCubeMessageExtractor
        try:
            extract_cls = pkg_resources.load_entry_point(
                distname, 'cubicweb.i18ncube', cubename)
        except (pkg_resources.DistributionNotFound, ImportError):
            pass  # no customization found
        print(underline_title('Updating i18n catalogs for cube %s' % cubename))
582
        chdir(cubedir)
583
        extractor = extract_cls(workdir, cubedir)
584
585
        potfile = extractor.generate_pot_file()
        if potfile is None:
586
            print('no message catalog for cube', cubename, 'nothing to translate')
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
            return ()
        print('-> merging main pot file with existing translations:', end=' ')
        chdir('i18n')
        toedit = []
        for lang in CubicWebNoAppConfiguration.cw_languages():
            print(lang, end=' ')
            cubepo = '%s.po' % lang
            if not osp.exists(cubepo):
                shutil.copy(potfile, cubepo)
            else:
                cmd = ['msgmerge', '-N', '-s', '-o', cubepo + 'new',
                       cubepo, potfile]
                execute2(cmd)
                ensure_fs_mode(cubepo)
                shutil.move('%snew' % cubepo, cubepo)
            toedit.append(osp.abspath(cubepo))
        print()
        return toedit
    finally:
        # cleanup
607
        shutil.rmtree(workdir)
608
609


610
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
611
612
613
    """Create a new cube.

    <cubename>
614
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
615
616
617
    """
    name = 'newcube'
    arguments = '<cubename>'
618
    min_args = max_args = 1
619
    options = (
620
621
622
623
624
625
626
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
627
628
629
630
631
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
632
633
634
635
636
637
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
656
657
658
659
660
661
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
662
663
        )

664
    LICENSES = {
665
666
667
668
669
670
671
672
673
674
675
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
676
677
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
678
''',
679

680
681
682
683
684
685
686
687
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
688
689
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
690
691
692
693
694
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
695
        }
696

Adrien Di Mascio's avatar
Adrien Di Mascio committed
697
    def run(self, args):
698
        import re
699
        from logilab.common.shellutils import ASK
700
701
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
702
703
            raise BadCommandUsage(
                'cube name must be a valid python module name')
704
        verbose = self.get('verbose')
705
706
        destdir = self.get('directory')
        if not destdir:
707
            cubespath = ServerConfiguration.cubes_search_path()
708
            if len(cubespath) > 1:
709
710
711
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
712
713
714
            destdir = cubespath[0]
        if not osp.isdir(destdir):
            print("-> creating cubes directory", destdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
715
            try:
716
                mkdir(destdir)
717
            except OSError as err:
718
                self.fail("failed to create directory %r\n(%s)"
719
                          % (destdir, err))
720
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
721
        if verbose:
722
            distname = input('Debian name for your cube ? [%s]): '
723
                             % default_name).strip()
724
            if not distname:
725
                distname = default_name
726
            elif not distname.startswith('cubicweb-'):
727
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
728
729
                    distname = 'cubicweb-' + distname
        else:
730
            distname = default_name
731
        if not re.match('[a-z][-a-z0-9]*$', distname):
732
733
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
734
735
736
737
        cubedir = osp.join(destdir, distname)
        if osp.exists(cubedir):
            self.fail("%s already exists!" % cubedir)
        skeldir = osp.join(BASEDIR, 'skeleton')
738
        longdesc = shortdesc = input(
739
            'Enter a short description for your cube: ')
740
        if verbose:
741
            longdesc = input(
742
                'Enter a long description (leave empty to reuse the short one): ')
743
744
745
746
        dependencies = {
            'six': '>= 1.4.0',
            'cubicweb': '>= %s' % cubicwebversion,
        }
747
        if verbose:
748
            dependencies.update(self._ask_for_dependencies())
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
        context = {
            'cubename': cubename,
            'distname': distname,
            'shortdesc': shortdesc,
            'longdesc': longdesc or shortdesc,
            'dependencies': dependencies,
            'version': cubicwebversion,
            'year': str(date.today().year),
            'author': self['author'],
            'author-email': self['author-email'],
            'rfc2822-date': datetime.now(tz=UTC).strftime('%a, %d %b %Y %T %z'),
            'author-web-site': self['author-web-site'],
            'license': self['license'],
            'long-license': self.LICENSES[self['license']],
        }
764
765
766
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
767
                        'cubes.*', 'uiprops.py*')
768
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
769

770
    def _ask_for_dependencies(self):
771
772
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
773
774
775
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
776
777
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
778
                depcubes.append(cube)
779
            if answer == 'type':
780
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
781
                break
782
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
783
                break
784
785
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
786

787
788
789
790

class ExamineLogCommand(Command):
    """Examine a rql log file.

791
    Will print out the following table
792

793
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
794

795
    sorted by descending cumulative time (clock). Time are expressed in seconds.
796

797
    Chances are the lines at the top are the ones that will bring the higher
798
    benefit after optimisation. Start there.
799
    """
800
    arguments = 'rql.log'
801
    name = 'exlog'
802
    options = ()
803

804
805
806
    def run(self, args):
        import re
        requests = {}
807
        for filepath in args:
808
            try:
Rémi Cardona's avatar
Rémi Cardona committed
809
                stream = open(filepath)
810
            except OSError as ex:
811
812
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
813
            for lineno, line in enumerate(stream):
814
815
816
817
818
819
820
821
822
823
824
825
826
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
827
                except Exception as exc:
828
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
829
        stat = []
830
        for rql, times in requests.items():
831
832
833
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
834
835
        stat.sort()
        stat.reverse()
836
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
837
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
838
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
839
840
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
841

842

843
844
845
846
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
847
    min_args = max_args = 1
848
849
    options = [
        ('output-file',
850
         {'type':'string', 'default': None,
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
884
885
886
887
888
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
889
        ]
890
891

    def run(self, args):
892
893
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
894
        from logilab.common.textutils import splitstrip
895
896
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
897
898
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
899
        cubes = splitstrip(args[0])
900
        dev_conf = DevConfiguration(*cubes)
901
902
903
904
905
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
906
907
908
909
910
911
912
913
914
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

931
932
933
934
        if viewer:
            p = Popen((viewer, out))
            p.wait()

935

936
for cmdcls in (UpdateCubicWebCatalogCommand,
937
               UpdateCubeCatalogCommand,
938
939
940
941
942
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)