devctl.py 33.5 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

Adrien Di Mascio's avatar
Adrien Di Mascio committed
23
24
__docformat__ = "restructuredtext en"

25
26
27
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
28
import sys
29
from datetime import datetime, date
30
from os import mkdir, chdir, path as osp
31
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
32

33
from pytz import UTC
34
35
from six.moves import input

Adrien Di Mascio's avatar
Adrien Di Mascio committed
36
from logilab.common import STD_BLACKLIST
37

Adrien Di Mascio's avatar
Adrien Di Mascio committed
38
from cubicweb.__pkginfo__ import version as cubicwebversion
39
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
40
from cubicweb.cwctl import CWCTL
41
from cubicweb.cwconfig import CubicWebNoAppConfiguration
42
43
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
44
45
46
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

47

48
49
50
51
52
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')


53
54
55
56
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
57
    creating = True
58
    cleanup_unused_appobjects = False
59
60
61
62
63

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
64

65
    def __init__(self, *cubes):
66
67
68
69
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
70
            self.load_site_cubicweb()
71
72
        else:
            self._cubes = ()
73

Adrien Di Mascio's avatar
Adrien Di Mascio committed
74
75
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
76
        return None
77
78
79
80

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
81
82
    def main_config_file(self):
        return None
83
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
84
        pass
85
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
86
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
87
88
    def default_log_file(self):
        return None
89
90
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
91
92
93
94


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
95
    for name, mod in list(sys.modules.items()):
Sylvain Thenault's avatar
Sylvain Thenault committed
96
97
98
99
100
101
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
102
103
104
        if mod.__file__ is None:
            # odd/rare but real
            continue
105
        for path in config.appobjects_path():
Sylvain Thenault's avatar
Sylvain Thenault committed
106
107
108
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
109

Adrien Di Mascio's avatar
Adrien Di Mascio committed
110
111
112
113
114
115
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
116
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
117
    if cubedir:
118
        cube = osp.split(cubedir)[-1]
119
120
        if cube.startswith('cubicweb_'):
            cube = cube[len('cubicweb_'):]
121
122
123
124
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
125
    else:
126
127
128
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
129
    schema = config.load_schema(remove_unused_rtypes=False)
130
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
131
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
132
133
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
134
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
135
136


137
def _generate_schema_pot(w, vreg, schema, libconfig=None):
138
    from cubicweb.i18n import add_msg
139
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
140
    w('# schema pot file, generated on %s\n'
141
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
142
143
144
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
145
    vregdone = set()
146
    afss = vreg['uicfg']['autoform_section']
147
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
148
    if libconfig is not None:
149
150
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
151
        from cubicweb.cwvreg import CWRegistryStore
152
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
153
        cleanup_sys_modules(libconfig)
154
        libvreg = CWRegistryStore(libconfig)
155
        libvreg.set_schema(libschema) # trigger objects registration
156
        libafss = libvreg['uicfg']['autoform_section']
157
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
158
159
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
160
161
162
163

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
164
    else:
165
        # processing cubicweb itself
166
        libschema = {}
167
168
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
169
170
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
171
    done = set()
172
173
174
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
175
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
176
        etype = eschema.type
177
178
179
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
180
            if not eschema.final:
181
                add_msg(w, 'This %s:' % etype)
182
                add_msg(w, 'New %s' % etype)
183
                add_msg(w, 'add a %s' % etype) # AddNewAction
184
185
186
187
188
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
189
190
191
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
192
        if eschema.final:
193
194
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
195
196
            if rschema.final:
                continue
197
            for tschema in targetschemas:
198

199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
226
227
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
228
    w('# subject and object forms for each relation type\n')
229
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
230
    w('\n')
231
232
233
234
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
235
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
236
        rtype = rschema.type
237
        if rtype not in libschema:
238
239
            # bw compat, necessary until all translation of relation are done
            # properly...
240
            add_msg(w, rtype)
241
            done.add(rtype)
242
243
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
244
            done.add(rschema.description)
245
246
247
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
248
        # add context information only for non-metadata rtypes
249
        if rschema not in NO_I18NCONTEXT:
250
            libsubjects = librschema and librschema.subjects() or ()
251
            for subjschema in rschema.subjects():
252
253
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
254
        if not (rschema.final or rschema.symmetric):
255
            if rschema not in NO_I18NCONTEXT:
256
257
258
259
260
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
261
262
                # bw compat, necessary until all translation of relation are
                # done properly...
263
                add_msg(w, '%s_object' % rtype)
264
        for rdef in rschema.rdefs.values():
265
266
267
268
269
270
271
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
272
    for objid in _iter_vreg_objids(vreg, vregdone):
273
274
275
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
276

277
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
278
    for reg, objdict in vreg.items():
279
280
        if reg in ('boxes', 'contentnavigation'):
            continue
281
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
282
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
283
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
284
                if objid in done:
285
                    break
286
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
287
                if pdefs:
288
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
289
                    done.add(objid)
290
                    break
291

292

Adrien Di Mascio's avatar
Adrien Di Mascio committed
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
310

Adrien Di Mascio's avatar
Adrien Di Mascio committed
311
312
313
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
314
    name = 'i18ncubicweb'
315
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
316
317
318
319

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
320
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
321
322
        import yams
        from logilab.common.fileutils import ensure_fs_mode
323
        from logilab.common.shellutils import globfind, find, rm
324
        from logilab.common.modutils import get_module_files
325
        from cubicweb.i18n import extract_from_tal, execute2
326
        tempdir = tempfile.mkdtemp(prefix='cw-')
327
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
328
329
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
330
331
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
332
333
334
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
335
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
336
337
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
338
        print('TAL', end=' ')
339
340
341
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
342
        print('-> generate .pot files.')
343
344
345
346
347
348
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
349
350
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
351
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
352
                                ]:
353
            potfile = osp.join(tempdir, '%s.pot' % id)
354
355
356
357
358
359
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
360
            if osp.exists(potfile):
361
362
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
363
364
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
365
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
366
367
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
368
        print('-> merging main pot file with existing translations.')
369
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
370
        toedit = []
371
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
372
            target = '%s.po' % lang
373
374
375
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
376
377
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
378
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
379
        # cleanup
380
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
381
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
382
383
384
385
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
386
387


388
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
389
390
391
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
392
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
393
    arguments = '[<cube>...]'
394

Adrien Di Mascio's avatar
Adrien Di Mascio committed
395
396
397
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
398
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
399
        else:
400
401
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
402
403
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
404
405
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
406

407

Adrien Di Mascio's avatar
Adrien Di Mascio committed
408
def update_cubes_catalogs(cubes):
409
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
410
    for cubedir in cubes:
411
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
412
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
413
            continue
414
        try:
415
            toedit = update_cube_catalogs(cubedir)
416
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
417
418
419
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
420
421
422
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
423
            print('*** error while updating catalogs for cube', cubedir)
424
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
425
426
        else:
            # instructions pour la suite
427
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
428
429
430
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
431
432
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
433
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
434

435
436
def update_cube_catalogs(cubedir):
    import shutil
437
    import tempfile
438
439
    from logilab.common.fileutils import ensure_fs_mode
    from logilab.common.shellutils import find, rm
440
    from cubicweb.i18n import extract_from_tal, execute2
441
    cube = osp.basename(osp.normpath(cubedir))
442
    tempdir = tempfile.mkdtemp()
Samuel Trégouët's avatar
Samuel Trégouët committed
443
    print(underline_title('Updating i18n catalogs for cube %s' % cube))
444
    chdir(cubedir)
445
    if osp.exists(osp.join('i18n', 'entities.pot')):
446
        warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'
447
448
449
450
             % osp.join('i18n', 'entities.pot'), DeprecationWarning)
        potfiles = [osp.join('i18n', 'entities.pot')]
    elif osp.exists(osp.join('i18n', 'static-messages.pot')):
        potfiles = [osp.join('i18n', 'static-messages.pot')]
451
452
    else:
        potfiles = []
Samuel Trégouët's avatar
Samuel Trégouët committed
453
454
    print('-> extracting messages:', end=' ')
    print('schema', end=' ')
455
    schemapot = osp.join(tempdir, 'schema.pot')
456
457
458
    potfiles.append(schemapot)
    # explicit close necessary else the file may not be yet flushed when
    # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
459
    schemapotstream = open(schemapot, 'w')
460
461
    generate_schema_pot(schemapotstream.write, cubedir)
    schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
462
    print('TAL', end=' ')
463
    tali18nfile = osp.join(tempdir, 'tali18n.py')
464
    ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST)
465
    extract_from_tal(ptfiles, tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
466
    print('Javascript')
467
468
    jsfiles =  [jsfile for jsfile in find('.', '.js')
                if osp.basename(jsfile).startswith('cub')]
469
    if jsfiles:
470
        tmppotfile = osp.join(tempdir, 'js.pot')
471
472
473
        cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-L', 'java',
               '--from-code=utf-8', '-o', tmppotfile] + jsfiles
        execute2(cmd)
474
        # no pot file created if there are no string to translate
475
        if osp.exists(tmppotfile):
476
            potfiles.append(tmppotfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
477
    print('-> creating cube-specific catalog')
478
    tmppotfile = osp.join(tempdir, 'generated.pot')
479
    cubefiles = find('.', '.py', blacklist=STD_BLACKLIST)
480
    cubefiles.append(tali18nfile)
481
482
483
    cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-o', tmppotfile]
    cmd.extend(cubefiles)
    execute2(cmd)
484
    if osp.exists(tmppotfile): # doesn't exists of no translation string found
485
        potfiles.append(tmppotfile)
486
    potfile = osp.join(tempdir, 'cube.pot')
Samuel Trégouët's avatar
Samuel Trégouët committed
487
    print('-> merging %i .pot files' % len(potfiles))
488
489
490
    cmd = ['msgcat', '-o', potfile]
    cmd.extend(potfiles)
    execute2(cmd)
491
    if not osp.exists(potfile):
Samuel Trégouët's avatar
Samuel Trégouët committed
492
        print('no message catalog for cube', cube, 'nothing to translate')
493
494
495
        # cleanup
        rm(tempdir)
        return ()
Samuel Trégouët's avatar
Samuel Trégouët committed
496
    print('-> merging main pot file with existing translations:', end=' ')
497
    chdir('i18n')
498
    toedit = []
499
    for lang in CubicWebNoAppConfiguration.cw_languages():
Samuel Trégouët's avatar
Samuel Trégouët committed
500
        print(lang, end=' ')
501
        cubepo = '%s.po' % lang
502
        if not osp.exists(cubepo):
503
504
            shutil.copy(potfile, cubepo)
        else:
505
506
            cmd = ['msgmerge','-N','-s','-o', cubepo+'new', cubepo, potfile]
            execute2(cmd)
507
508
            ensure_fs_mode(cubepo)
            shutil.move('%snew' % cubepo, cubepo)
509
        toedit.append(osp.abspath(cubepo))
Samuel Trégouët's avatar
Samuel Trégouët committed
510
    print()
511
512
513
514
515
    # cleanup
    rm(tempdir)
    return toedit


516
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
517
518
519
    """Create a new cube.

    <cubename>
520
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
521
522
523
    """
    name = 'newcube'
    arguments = '<cubename>'
524
    min_args = max_args = 1
525
    options = (
526
527
528
529
530
531
532
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
533
534
535
536
537
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
538
539
540
541
542
543
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
562
563
564
565
566
567
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
568
569
        )

570
    LICENSES = {
571
572
573
574
575
576
577
578
579
580
581
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
582
583
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
584
''',
585

586
587
588
589
590
591
592
593
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
594
595
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
596
597
598
599
600
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
601
        }
602

Adrien Di Mascio's avatar
Adrien Di Mascio committed
603
    def run(self, args):
604
        import re
605
        from logilab.common.shellutils import ASK
606
607
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
608
609
            raise BadCommandUsage(
                'cube name must be a valid python module name')
610
        verbose = self.get('verbose')
611
612
        destdir = self.get('directory')
        if not destdir:
613
            cubespath = ServerConfiguration.cubes_search_path()
614
            if len(cubespath) > 1:
615
616
617
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
618
619
620
            destdir = cubespath[0]
        if not osp.isdir(destdir):
            print("-> creating cubes directory", destdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
621
            try:
622
                mkdir(destdir)
623
            except OSError as err:
624
                self.fail("failed to create directory %r\n(%s)"
625
                          % (destdir, err))
626
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
627
        if verbose:
628
            distname = input('Debian name for your cube ? [%s]): '
629
                             % default_name).strip()
630
            if not distname:
631
                distname = default_name
632
            elif not distname.startswith('cubicweb-'):
633
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
634
635
                    distname = 'cubicweb-' + distname
        else:
636
            distname = default_name
637
        if not re.match('[a-z][-a-z0-9]*$', distname):
638
639
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
640
641
642
643
        cubedir = osp.join(destdir, distname)
        if osp.exists(cubedir):
            self.fail("%s already exists!" % cubedir)
        skeldir = osp.join(BASEDIR, 'skeleton')
644
        longdesc = shortdesc = input(
645
            'Enter a short description for your cube: ')
646
        if verbose:
647
            longdesc = input(
648
                'Enter a long description (leave empty to reuse the short one): ')
649
650
651
652
        dependencies = {
            'six': '>= 1.4.0',
            'cubicweb': '>= %s' % cubicwebversion,
        }
653
        if verbose:
654
            dependencies.update(self._ask_for_dependencies())
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
        context = {
            'cubename': cubename,
            'distname': distname,
            'shortdesc': shortdesc,
            'longdesc': longdesc or shortdesc,
            'dependencies': dependencies,
            'version': cubicwebversion,
            'year': str(date.today().year),
            'author': self['author'],
            'author-email': self['author-email'],
            'rfc2822-date': datetime.now(tz=UTC).strftime('%a, %d %b %Y %T %z'),
            'author-web-site': self['author-web-site'],
            'license': self['license'],
            'long-license': self.LICENSES[self['license']],
        }
670
671
672
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
673
                        'cubes.*', 'uiprops.py*')
674
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
675

676
    def _ask_for_dependencies(self):
677
678
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
679
680
681
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
682
683
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
684
                depcubes.append(cube)
685
            if answer == 'type':
686
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
687
                break
688
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
689
                break
690
691
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
692

693
694
695
696

class ExamineLogCommand(Command):
    """Examine a rql log file.

697
    Will print out the following table
698

699
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
700

701
    sorted by descending cumulative time (clock). Time are expressed in seconds.
702

703
    Chances are the lines at the top are the ones that will bring the higher
704
    benefit after optimisation. Start there.
705
    """
706
    arguments = 'rql.log'
707
    name = 'exlog'
708
    options = ()
709

710
711
712
    def run(self, args):
        import re
        requests = {}
713
        for filepath in args:
714
            try:
Rémi Cardona's avatar
Rémi Cardona committed
715
                stream = open(filepath)
716
            except OSError as ex:
717
718
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
719
            for lineno, line in enumerate(stream):
720
721
722
723
724
725
726
727
728
729
730
731
732
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
733
                except Exception as exc:
734
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
735
        stat = []
736
        for rql, times in requests.items():
737
738
739
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
740
741
        stat.sort()
        stat.reverse()
742
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
743
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
744
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
745
746
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
747

748

749
750
751
752
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
753
    min_args = max_args = 1
754
755
    options = [
        ('output-file',
756
         {'type':'string', 'default': None,
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
790
791
792
793
794
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
795
        ]
796
797

    def run(self, args):
798
799
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
800
        from logilab.common.textutils import splitstrip
801
802
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
803
804
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
805
        cubes = splitstrip(args[0])
806
        dev_conf = DevConfiguration(*cubes)
807
808
809
810
811
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
812
813
814
815
816
817
818
819
820
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

837
838
839
840
        if viewer:
            p = Popen((viewer, out))
            p.wait()

841

842
for cmdcls in (UpdateCubicWebCatalogCommand,
843
               UpdateCubeCatalogCommand,
844
845
846
847
848
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)