devctl.py 35.9 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
21
22
23
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
24
25
26

import shutil
import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
27
import sys
28
from datetime import datetime, date
29
from os import getcwd, mkdir, chdir, path as osp
30
import pkg_resources
Adrien Di Mascio's avatar
Adrien Di Mascio committed
31

32
from pytz import UTC
33

Adrien Di Mascio's avatar
Adrien Di Mascio committed
34
from logilab.common import STD_BLACKLIST
35
from logilab.common.modutils import clean_sys_modules
36
37
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find
38

Adrien Di Mascio's avatar
Adrien Di Mascio committed
39
from cubicweb.__pkginfo__ import version as cubicwebversion
40
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
41
from cubicweb.i18n import extract_from_tal, execute2
42
from cubicweb.cwctl import CWCTL
43
from cubicweb.cwconfig import CubicWebNoAppConfiguration
44
45
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
46
47
48
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

49

50
51
52
__docformat__ = "restructuredtext en"


53
54
55
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')
56
STD_BLACKLIST.add('node_modules')
57
58


59
60
61
62
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
63
    creating = True
64
    cleanup_unused_appobjects = False
65
66
67
68
69

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
70

71
    def __init__(self, *cubes):
72
73
74
75
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
76
            self.load_site_cubicweb()
77
78
        else:
            self._cubes = ()
79

Adrien Di Mascio's avatar
Adrien Di Mascio committed
80
81
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
82
        return None
83
84
85
86

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
87
88
    def main_config_file(self):
        return None
89
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
90
        pass
91
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
92
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
93
94
    def default_log_file(self):
        return None
95
96
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
97
98


Adrien Di Mascio's avatar
Adrien Di Mascio committed
99
100
101
102
103
104
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
105
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
106
    if cubedir:
107
        cube = osp.split(cubedir)[-1]
108
109
        if cube.startswith('cubicweb_'):
            cube = cube[len('cubicweb_'):]
110
111
112
113
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
114
    else:
115
116
        config = DevConfiguration()
        cube = libconfig = None
117
    clean_sys_modules(config.appobjects_modnames())
118
    schema = config.load_schema(remove_unused_rtypes=False)
119
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
120
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
121
122
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
123
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
124
125


126
def _generate_schema_pot(w, vreg, schema, libconfig=None):
127
    from cubicweb.i18n import add_msg
128
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
129
    w('# schema pot file, generated on %s\n'
130
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
131
132
133
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
134
    vregdone = set()
135
    afss = vreg['uicfg']['autoform_section']
136
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
137
    if libconfig is not None:
138
139
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
140
        from cubicweb.cwvreg import CWRegistryStore
141
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
142
        clean_sys_modules(libconfig.appobjects_modnames())
143
        libvreg = CWRegistryStore(libconfig)
144
        libvreg.set_schema(libschema) # trigger objects registration
145
        libafss = libvreg['uicfg']['autoform_section']
146
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
147
148
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
149
150
151
152

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
153
    else:
154
        # processing cubicweb itself
155
        libschema = {}
156
157
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
158
159
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
160
    done = set()
161
162
163
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
164
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
165
        etype = eschema.type
166
167
168
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
169
            if not eschema.final:
170
                add_msg(w, 'This %s:' % etype)
171
                add_msg(w, 'New %s' % etype)
172
                add_msg(w, 'add a %s' % etype) # AddNewAction
173
174
175
176
177
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
178
179
180
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
181
        if eschema.final:
182
183
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
184
185
            if rschema.final:
                continue
186
            for tschema in targetschemas:
187

188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
215
216
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
217
    w('# subject and object forms for each relation type\n')
218
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
219
    w('\n')
220
221
222
223
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
224
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
225
        rtype = rschema.type
226
        if rtype not in libschema:
227
228
            # bw compat, necessary until all translation of relation are done
            # properly...
229
            add_msg(w, rtype)
230
            done.add(rtype)
231
232
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
233
            done.add(rschema.description)
234
235
236
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
237
        # add context information only for non-metadata rtypes
238
        if rschema not in NO_I18NCONTEXT:
239
            libsubjects = librschema and librschema.subjects() or ()
240
            for subjschema in rschema.subjects():
241
242
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
243
        if not (rschema.final or rschema.symmetric):
244
            if rschema not in NO_I18NCONTEXT:
245
246
247
248
249
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
250
251
                # bw compat, necessary until all translation of relation are
                # done properly...
252
                add_msg(w, '%s_object' % rtype)
253
        for rdef in rschema.rdefs.values():
254
255
256
257
258
259
260
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
261
    for objid in _iter_vreg_objids(vreg, vregdone):
262
263
264
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
265

266
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
267
    for reg, objdict in vreg.items():
268
269
        if reg in ('boxes', 'contentnavigation'):
            continue
270
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
271
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
272
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
273
                if objid in done:
274
                    break
275
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
276
                if pdefs:
277
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
278
                    done.add(objid)
279
                    break
280

281

Adrien Di Mascio's avatar
Adrien Di Mascio committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
299

Adrien Di Mascio's avatar
Adrien Di Mascio committed
300
301
302
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
303
    name = 'i18ncubicweb'
304
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
305
306
307
308

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
309
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
310
311
        import yams
        from logilab.common.fileutils import ensure_fs_mode
312
        from logilab.common.shellutils import globfind, find, rm
313
        from logilab.common.modutils import get_module_files
314
        from cubicweb.i18n import extract_from_tal, execute2
315
        tempdir = tempfile.mkdtemp(prefix='cw-')
316
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
317
318
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
319
320
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
321
322
323
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
324
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
325
326
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
327
        print('TAL', end=' ')
328
329
330
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
331
        print('-> generate .pot files.')
332
333
334
335
336
337
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
338
339
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
340
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
341
                                ]:
342
            potfile = osp.join(tempdir, '%s.pot' % id)
343
            cmd = ['xgettext', '--from-code=UTF-8', '--no-location', '--omit-header', '-k_']
344
345
346
347
348
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
349
            if osp.exists(potfile):
350
351
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
352
353
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
354
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
355
356
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
357
        print('-> merging main pot file with existing translations.')
358
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
359
        toedit = []
360
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
361
            target = '%s.po' % lang
362
363
364
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
365
366
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
367
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
368
        # cleanup
369
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
370
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
371
372
373
374
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
375
376


377
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
378
379
380
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
381
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
382
    arguments = '[<cube>...]'
383

Adrien Di Mascio's avatar
Adrien Di Mascio committed
384
385
386
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
387
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
388
        else:
389
390
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
391
392
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
393
394
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
395

396

Adrien Di Mascio's avatar
Adrien Di Mascio committed
397
def update_cubes_catalogs(cubes):
398
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
399
    for cubedir in cubes:
400
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
401
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
402
            continue
403
        try:
404
            toedit = update_cube_catalogs(cubedir)
405
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
406
407
408
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
409
410
411
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
412
            print('*** error while updating catalogs for cube', cubedir)
413
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
414
415
        else:
            # instructions pour la suite
416
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
417
418
419
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
420
421
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
422
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
423

424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476

class I18nCubeMessageExtractor(object):
    """This class encapsulates all the xgettext extraction logic

    ``generate_pot_file`` is the main entry point called by the ``i18ncube``
    command. A cube might decide to customize extractors to ignore a given
    directory or to extract messages from a new file type (e.g. .jinja2 files)

    For each file type, the class must define two methods:

    - ``collect_{filetype}()`` that must return the list of files
      xgettext should inspect,

    - ``extract_{filetype}(files)`` that calls xgettext and returns the
      path to the generated ``pot`` file
    """
    blacklist = STD_BLACKLIST
    formats = ['tal', 'js', 'py']

    def __init__(self, workdir, cubedir):
        self.workdir = workdir
        self.cubedir = cubedir

    def generate_pot_file(self):
        """main entry point: return the generated ``cube.pot`` file

        This function first generates all the pot files (schema, tal,
        py, js) and then merges them in a single ``cube.pot`` that will
        be used to eventually update the ``i18n/*.po`` files.
        """
        potfiles = self.generate_pot_files()
        potfile = osp.join(self.workdir, 'cube.pot')
        print('-> merging %i .pot files' % len(potfiles))
        cmd = ['msgcat', '-o', potfile]
        cmd.extend(potfiles)
        execute2(cmd)
        return potfile if osp.exists(potfile) else None

    def find(self, exts, blacklist=None):
        """collect files with extensions ``exts`` in the cube directory
        """
        if blacklist is None:
            blacklist = self.blacklist
        return find(self.cubedir, exts, blacklist=blacklist)

    def generate_pot_files(self):
        """generate and return the list of all ``pot`` files for the cube

        - static-messages.pot,
        - schema.pot,
        - one ``pot`` file for each inspected format (.py, .js, etc.)
        """
        print('-> extracting messages:', end=' ')
477
        potfiles = []
478
        # static messages
Denis Laxalde's avatar
Denis Laxalde committed
479
        if osp.exists(osp.join('i18n', 'static-messages.pot')):
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
            potfiles.append(osp.join('i18n', 'static-messages.pot'))
        # messages from schema
        potfiles.append(self.schemapot())
        # messages from sourcecode
        for fmt in self.formats:
            collector = getattr(self, 'collect_{0}'.format(fmt))
            extractor = getattr(self, 'extract_{0}'.format(fmt))
            files = collector()
            if files:
                potfile = extractor(files)
                if potfile:
                    potfiles.append(potfile)
        return potfiles

    def schemapot(self):
        """generate the ``schema.pot`` file"""
        schemapot = osp.join(self.workdir, 'schema.pot')
        print('schema', end=' ')
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
        schemapotstream = open(schemapot, 'w')
        generate_schema_pot(schemapotstream.write, self.cubedir)
        schemapotstream.close()
        return schemapot

    def _xgettext(self, files, output, k='_', extraopts=''):
        """shortcut to execute the xgettext command and return output file
        """
        tmppotfile = osp.join(self.workdir, output)
509
510
        cmd = ['xgettext', '--from-code=UTF-8', '--no-location', '--omit-header',
               '-k' + k, '-o', tmppotfile] + extraopts.split() + files
511
        execute2(cmd)
512
        if osp.exists(tmppotfile):
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
            return tmppotfile

    def collect_tal(self):
        print('TAL', end=' ')
        return self.find(('.py', '.pt'))

    def extract_tal(self, files):
        tali18nfile = osp.join(self.workdir, 'tali18n.py')
        extract_from_tal(files, tali18nfile)
        return self._xgettext(files, output='tal.pot')

    def collect_js(self):
        print('Javascript')
        return [jsfile for jsfile in self.find('.js')
                if osp.basename(jsfile).startswith('cub')]

    def extract_js(self, files):
        return self._xgettext(files, output='js.pot',
                              extraopts='-L java --from-code=utf-8')

    def collect_py(self):
        print('-> creating cube-specific catalog')
        return self.find('.py')

    def extract_py(self, files):
        return self._xgettext(files, output='py.pot')


def update_cube_catalogs(cubedir):
    cubedir = osp.abspath(osp.normpath(cubedir))
    workdir = tempfile.mkdtemp()
544
    try:
545
546
547
548
549
550
        cubename = osp.basename(cubedir)
        if cubename.startswith('cubicweb_'):  # new layout
            distname = cubename
            cubename = cubename[len('cubicweb_'):]
        else:
            distname = 'cubicweb_' + cubename
551
        print('cubedir', cubedir)
552
553
554
555
556
557
558
        extract_cls = I18nCubeMessageExtractor
        try:
            extract_cls = pkg_resources.load_entry_point(
                distname, 'cubicweb.i18ncube', cubename)
        except (pkg_resources.DistributionNotFound, ImportError):
            pass  # no customization found
        print(underline_title('Updating i18n catalogs for cube %s' % cubename))
559
        chdir(cubedir)
560
        extractor = extract_cls(workdir, cubedir)
561
562
        potfile = extractor.generate_pot_file()
        if potfile is None:
563
            print('no message catalog for cube', cubename, 'nothing to translate')
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
            return ()
        print('-> merging main pot file with existing translations:', end=' ')
        chdir('i18n')
        toedit = []
        for lang in CubicWebNoAppConfiguration.cw_languages():
            print(lang, end=' ')
            cubepo = '%s.po' % lang
            if not osp.exists(cubepo):
                shutil.copy(potfile, cubepo)
            else:
                cmd = ['msgmerge', '-N', '-s', '-o', cubepo + 'new',
                       cubepo, potfile]
                execute2(cmd)
                ensure_fs_mode(cubepo)
                shutil.move('%snew' % cubepo, cubepo)
            toedit.append(osp.abspath(cubepo))
        print()
        return toedit
    finally:
        # cleanup
584
        shutil.rmtree(workdir)
585
586


587
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
588
589
590
    """Create a new cube.

    <cubename>
591
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
592
593
594
    """
    name = 'newcube'
    arguments = '<cubename>'
595
    min_args = max_args = 1
596
    options = (
597
598
599
600
601
602
603
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
604
605
606
607
608
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
609
610
611
612
613
614
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
615
616
617
618
619
        ("short-description",
         {'short': 's', 'type': 'string', 'metavar': '<short description>',
          'help': 'short description for the cube',
         }
         ),
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
638
639
640
641
642
643
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
644
645
        )

646
    LICENSES = {
647
648
649
650
651
652
653
654
655
656
657
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
658
659
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
660
''',
661

662
663
664
665
666
667
668
669
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
670
671
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
672
673
674
675
676
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
677
        }
678

Adrien Di Mascio's avatar
Adrien Di Mascio committed
679
    def run(self, args):
680
        import re
681
        from logilab.common.shellutils import ASK
682
683
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
684
685
            raise BadCommandUsage(
                'cube name must be a valid python module name')
686
        verbose = self.get('verbose')
687
688
        destdir = self.get('directory')
        if not destdir:
689
690
            destdir = getcwd()
        elif not osp.isdir(destdir):
691
            print("-> creating cubes directory", destdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
692
            try:
693
                mkdir(destdir)
694
            except OSError as err:
695
                self.fail("failed to create directory %r\n(%s)"
696
                          % (destdir, err))
697
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
698
        if verbose:
699
            distname = input('Debian name for your cube ? [%s]): '
700
                             % default_name).strip()
701
            if not distname:
702
                distname = default_name
703
            elif not distname.startswith('cubicweb-'):
704
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
705
706
                    distname = 'cubicweb-' + distname
        else:
707
            distname = default_name
708
        if not re.match('[a-z][-a-z0-9]*$', distname):
709
710
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
711
712
713
714
        cubedir = osp.join(destdir, distname)
        if osp.exists(cubedir):
            self.fail("%s already exists!" % cubedir)
        skeldir = osp.join(BASEDIR, 'skeleton')
715
        longdesc = shortdesc = self['short-description'] or input(
716
            'Enter a short description for your cube: ')
717
        if verbose:
718
            longdesc = input(
719
                'Enter a long description (leave empty to reuse the short one): ')
720
721
722
        dependencies = {
            'cubicweb': '>= %s' % cubicwebversion,
        }
723
        if verbose:
724
            dependencies.update(self._ask_for_dependencies())
725
726
727
728
729
        context = {
            'cubename': cubename,
            'distname': distname,
            'shortdesc': shortdesc,
            'longdesc': longdesc or shortdesc,
730
            'dependencies': str(dependencies).replace("'", '"'),  # to respect black formatting
731
732
733
734
735
736
737
738
739
            'version': cubicwebversion,
            'year': str(date.today().year),
            'author': self['author'],
            'author-email': self['author-email'],
            'rfc2822-date': datetime.now(tz=UTC).strftime('%a, %d %b %Y %T %z'),
            'author-web-site': self['author-web-site'],
            'license': self['license'],
            'long-license': self.LICENSES[self['license']],
        }
740
741
742
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
743
                        'cubes.*', 'uiprops.py*')
744
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
745

746
    def _ask_for_dependencies(self):
747
748
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
749
750
751
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
752
753
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
754
                depcubes.append(cube)
755
            if answer == 'type':
756
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
757
                break
758
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
759
                break
760
761
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
762

763
764
765
766

class ExamineLogCommand(Command):
    """Examine a rql log file.

767
    Will print out the following table
768

769
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
770

771
    sorted by descending cumulative time (clock). Time are expressed in seconds.
772

773
    Chances are the lines at the top are the ones that will bring the higher
774
    benefit after optimisation. Start there.
775
    """
776
    arguments = 'rql.log'
777
    name = 'exlog'
778
    options = ()
779

780
781
782
    def run(self, args):
        import re
        requests = {}
783
        for filepath in args:
784
            try:
Rémi Cardona's avatar
Rémi Cardona committed
785
                stream = open(filepath)
786
            except OSError as ex:
787
788
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
789
            for lineno, line in enumerate(stream):
790
791
792
793
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
794
                    rql = re.sub(r"(\'\w+': \d*)", '', rql)
795
796
797
798
799
800
801
802
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
803
                except Exception as exc:
804
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
805
        stat = []
806
        for rql, times in requests.items():
807
808
809
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
810
811
        stat.sort()
        stat.reverse()
812
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
813
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
814
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
815
816
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
817

818

819
820
821
822
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
823
    min_args = max_args = 1
824
825
    options = [
        ('output-file',
826
         {'type':'string', 'default': None,
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
860
861
862
863
864
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
865
        ]
866
867

    def run(self, args):
868
869
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
870
        from logilab.common.textutils import splitstrip
871
872
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
873
874
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
875
        cubes = splitstrip(args[0])
876
        dev_conf = DevConfiguration(*cubes)
877
878
879
880
881
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
882
883
884
885
886
887
888
889
890
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

907
908
909
910
        if viewer:
            p = Popen((viewer, out))
            p.wait()

911

912
for cmdcls in (UpdateCubicWebCatalogCommand,
913
               UpdateCubeCatalogCommand,
914
915
916
917
918
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)