devctl.py 33.5 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

Adrien Di Mascio's avatar
Adrien Di Mascio committed
23
24
__docformat__ = "restructuredtext en"

25
26
27
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
28
import sys
29
from datetime import datetime, date
30
from os import mkdir, chdir, path as osp
31
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
32

33
34
from six.moves import input

Adrien Di Mascio's avatar
Adrien Di Mascio committed
35
from logilab.common import STD_BLACKLIST
36

Adrien Di Mascio's avatar
Adrien Di Mascio committed
37
from cubicweb.__pkginfo__ import version as cubicwebversion
38
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
39
from cubicweb.cwctl import CWCTL
40
from cubicweb.cwconfig import CubicWebNoAppConfiguration
41
42
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
43
44
45
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

46

47
48
49
50
51
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')


52
53
54
55
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
56
    creating = True
57
    cleanup_unused_appobjects = False
58
59
60
61
62

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
63

64
    def __init__(self, *cubes):
65
66
67
68
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
69
            self.load_site_cubicweb()
70
71
        else:
            self._cubes = ()
72

Adrien Di Mascio's avatar
Adrien Di Mascio committed
73
74
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
75
        return None
76
77
78
79

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
80
81
    def main_config_file(self):
        return None
82
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
83
        pass
84
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
85
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
86
87
    def default_log_file(self):
        return None
88
89
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
90
91
92
93


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
94
    for name, mod in list(sys.modules.items()):
Sylvain Thenault's avatar
Sylvain Thenault committed
95
96
97
98
99
100
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
101
102
103
        if mod.__file__ is None:
            # odd/rare but real
            continue
104
        for path in config.appobjects_path():
Sylvain Thenault's avatar
Sylvain Thenault committed
105
106
107
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
108

Adrien Di Mascio's avatar
Adrien Di Mascio committed
109
110
111
112
113
114
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
115
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
116
    if cubedir:
117
        cube = osp.split(cubedir)[-1]
118
119
        if cube.startswith('cubicweb_'):
            cube = cube[len('cubicweb_'):]
120
121
122
123
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
124
    else:
125
126
127
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
128
    schema = config.load_schema(remove_unused_rtypes=False)
129
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
130
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
131
132
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
133
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
134
135


136
def _generate_schema_pot(w, vreg, schema, libconfig=None):
137
    from cubicweb.i18n import add_msg
138
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
139
    w('# schema pot file, generated on %s\n'
140
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
141
142
143
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
144
    vregdone = set()
145
    afss = vreg['uicfg']['autoform_section']
146
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
147
    if libconfig is not None:
148
149
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
150
        from cubicweb.cwvreg import CWRegistryStore
151
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
152
        cleanup_sys_modules(libconfig)
153
        libvreg = CWRegistryStore(libconfig)
154
        libvreg.set_schema(libschema) # trigger objects registration
155
        libafss = libvreg['uicfg']['autoform_section']
156
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
157
158
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
159
160
161
162

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
163
    else:
164
        # processing cubicweb itself
165
        libschema = {}
166
167
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
168
169
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
170
    done = set()
171
172
173
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
174
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
175
        etype = eschema.type
176
177
178
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
179
            if not eschema.final:
180
                add_msg(w, 'This %s:' % etype)
181
                add_msg(w, 'New %s' % etype)
182
                add_msg(w, 'add a %s' % etype) # AddNewAction
183
184
185
186
187
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
188
189
190
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
191
        if eschema.final:
192
193
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
194
195
            if rschema.final:
                continue
196
            for tschema in targetschemas:
197

198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
225
226
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
227
    w('# subject and object forms for each relation type\n')
228
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
229
    w('\n')
230
231
232
233
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
234
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
235
        rtype = rschema.type
236
        if rtype not in libschema:
237
238
            # bw compat, necessary until all translation of relation are done
            # properly...
239
            add_msg(w, rtype)
240
            done.add(rtype)
241
242
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
243
            done.add(rschema.description)
244
245
246
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
247
        # add context information only for non-metadata rtypes
248
        if rschema not in NO_I18NCONTEXT:
249
            libsubjects = librschema and librschema.subjects() or ()
250
            for subjschema in rschema.subjects():
251
252
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
253
        if not (rschema.final or rschema.symmetric):
254
            if rschema not in NO_I18NCONTEXT:
255
256
257
258
259
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
260
261
                # bw compat, necessary until all translation of relation are
                # done properly...
262
                add_msg(w, '%s_object' % rtype)
263
        for rdef in rschema.rdefs.values():
264
265
266
267
268
269
270
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
271
    for objid in _iter_vreg_objids(vreg, vregdone):
272
273
274
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
275

276
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
277
    for reg, objdict in vreg.items():
278
279
        if reg in ('boxes', 'contentnavigation'):
            continue
280
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
281
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
282
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
283
                if objid in done:
284
                    break
285
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
286
                if pdefs:
287
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
288
                    done.add(objid)
289
                    break
290

291

Adrien Di Mascio's avatar
Adrien Di Mascio committed
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
309

Adrien Di Mascio's avatar
Adrien Di Mascio committed
310
311
312
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
313
    name = 'i18ncubicweb'
314
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
315
316
317
318

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
319
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
320
321
        import yams
        from logilab.common.fileutils import ensure_fs_mode
322
        from logilab.common.shellutils import globfind, find, rm
323
        from logilab.common.modutils import get_module_files
324
        from cubicweb.i18n import extract_from_tal, execute2
325
        tempdir = tempfile.mkdtemp(prefix='cw-')
326
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
327
328
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
329
330
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
331
332
333
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
334
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
335
336
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
337
        print('TAL', end=' ')
338
339
340
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
341
        print('-> generate .pot files.')
342
343
344
345
346
347
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
348
349
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
350
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
351
                                ]:
352
            potfile = osp.join(tempdir, '%s.pot' % id)
353
354
355
356
357
358
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
359
            if osp.exists(potfile):
360
361
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
362
363
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
364
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
365
366
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
367
        print('-> merging main pot file with existing translations.')
368
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
369
        toedit = []
370
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
371
            target = '%s.po' % lang
372
373
374
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
375
376
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
377
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
378
        # cleanup
379
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
380
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
381
382
383
384
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
385
386


387
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
388
389
390
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
391
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
392
    arguments = '[<cube>...]'
393

Adrien Di Mascio's avatar
Adrien Di Mascio committed
394
395
396
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
397
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
398
        else:
399
400
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
401
402
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
403
404
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
405

406

Adrien Di Mascio's avatar
Adrien Di Mascio committed
407
def update_cubes_catalogs(cubes):
408
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
409
    for cubedir in cubes:
410
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
411
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
412
            continue
413
        try:
414
            toedit = update_cube_catalogs(cubedir)
415
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
416
417
418
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
419
420
421
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
422
            print('*** error while updating catalogs for cube', cubedir)
423
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
424
425
        else:
            # instructions pour la suite
426
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
427
428
429
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
430
431
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
432
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
433

434
435
def update_cube_catalogs(cubedir):
    import shutil
436
    import tempfile
437
438
    from logilab.common.fileutils import ensure_fs_mode
    from logilab.common.shellutils import find, rm
439
    from cubicweb.i18n import extract_from_tal, execute2
440
    cube = osp.basename(osp.normpath(cubedir))
441
    tempdir = tempfile.mkdtemp()
Samuel Trégouët's avatar
Samuel Trégouët committed
442
    print(underline_title('Updating i18n catalogs for cube %s' % cube))
443
    chdir(cubedir)
444
    if osp.exists(osp.join('i18n', 'entities.pot')):
445
        warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'
446
447
448
449
             % osp.join('i18n', 'entities.pot'), DeprecationWarning)
        potfiles = [osp.join('i18n', 'entities.pot')]
    elif osp.exists(osp.join('i18n', 'static-messages.pot')):
        potfiles = [osp.join('i18n', 'static-messages.pot')]
450
451
    else:
        potfiles = []
Samuel Trégouët's avatar
Samuel Trégouët committed
452
453
    print('-> extracting messages:', end=' ')
    print('schema', end=' ')
454
    schemapot = osp.join(tempdir, 'schema.pot')
455
456
457
    potfiles.append(schemapot)
    # explicit close necessary else the file may not be yet flushed when
    # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
458
    schemapotstream = open(schemapot, 'w')
459
460
    generate_schema_pot(schemapotstream.write, cubedir)
    schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
461
    print('TAL', end=' ')
462
    tali18nfile = osp.join(tempdir, 'tali18n.py')
463
    ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST)
464
    extract_from_tal(ptfiles, tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
465
    print('Javascript')
466
467
    jsfiles =  [jsfile for jsfile in find('.', '.js')
                if osp.basename(jsfile).startswith('cub')]
468
    if jsfiles:
469
        tmppotfile = osp.join(tempdir, 'js.pot')
470
471
472
        cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-L', 'java',
               '--from-code=utf-8', '-o', tmppotfile] + jsfiles
        execute2(cmd)
473
        # no pot file created if there are no string to translate
474
        if osp.exists(tmppotfile):
475
            potfiles.append(tmppotfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
476
    print('-> creating cube-specific catalog')
477
    tmppotfile = osp.join(tempdir, 'generated.pot')
478
    cubefiles = find('.', '.py', blacklist=STD_BLACKLIST)
479
    cubefiles.append(tali18nfile)
480
481
482
    cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-o', tmppotfile]
    cmd.extend(cubefiles)
    execute2(cmd)
483
    if osp.exists(tmppotfile): # doesn't exists of no translation string found
484
        potfiles.append(tmppotfile)
485
    potfile = osp.join(tempdir, 'cube.pot')
Samuel Trégouët's avatar
Samuel Trégouët committed
486
    print('-> merging %i .pot files' % len(potfiles))
487
488
489
    cmd = ['msgcat', '-o', potfile]
    cmd.extend(potfiles)
    execute2(cmd)
490
    if not osp.exists(potfile):
Samuel Trégouët's avatar
Samuel Trégouët committed
491
        print('no message catalog for cube', cube, 'nothing to translate')
492
493
494
        # cleanup
        rm(tempdir)
        return ()
Samuel Trégouët's avatar
Samuel Trégouët committed
495
    print('-> merging main pot file with existing translations:', end=' ')
496
    chdir('i18n')
497
    toedit = []
498
    for lang in CubicWebNoAppConfiguration.cw_languages():
Samuel Trégouët's avatar
Samuel Trégouët committed
499
        print(lang, end=' ')
500
        cubepo = '%s.po' % lang
501
        if not osp.exists(cubepo):
502
503
            shutil.copy(potfile, cubepo)
        else:
504
505
            cmd = ['msgmerge','-N','-s','-o', cubepo+'new', cubepo, potfile]
            execute2(cmd)
506
507
            ensure_fs_mode(cubepo)
            shutil.move('%snew' % cubepo, cubepo)
508
        toedit.append(osp.abspath(cubepo))
Samuel Trégouët's avatar
Samuel Trégouët committed
509
    print()
510
511
512
513
514
    # cleanup
    rm(tempdir)
    return toedit


515
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
516
517
518
    """Create a new cube.

    <cubename>
519
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
520
521
522
    """
    name = 'newcube'
    arguments = '<cubename>'
523
    min_args = max_args = 1
524
    options = (
525
526
527
528
529
530
531
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
532
533
534
535
536
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
537
538
539
540
541
542
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
561
562
563
564
565
566
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
567
568
        )

569
    LICENSES = {
570
571
572
573
574
575
576
577
578
579
580
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
581
582
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
583
''',
584

585
586
587
588
589
590
591
592
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
593
594
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
595
596
597
598
599
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
600
        }
601

Adrien Di Mascio's avatar
Adrien Di Mascio committed
602
    def run(self, args):
603
        import re
604
        from logilab.common.shellutils import ASK
605
606
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
607
608
            raise BadCommandUsage(
                'cube name must be a valid python module name')
609
        verbose = self.get('verbose')
610
611
        destdir = self.get('directory')
        if not destdir:
612
            cubespath = ServerConfiguration.cubes_search_path()
613
            if len(cubespath) > 1:
614
615
616
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
617
618
619
            destdir = cubespath[0]
        if not osp.isdir(destdir):
            print("-> creating cubes directory", destdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
620
            try:
621
                mkdir(destdir)
622
            except OSError as err:
623
                self.fail("failed to create directory %r\n(%s)"
624
                          % (destdir, err))
625
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
626
        if verbose:
627
            distname = input('Debian name for your cube ? [%s]): '
628
                             % default_name).strip()
629
            if not distname:
630
                distname = default_name
631
            elif not distname.startswith('cubicweb-'):
632
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
633
634
                    distname = 'cubicweb-' + distname
        else:
635
            distname = default_name
636
        if not re.match('[a-z][-a-z0-9]*$', distname):
637
638
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
639
640
641
642
        cubedir = osp.join(destdir, distname)
        if osp.exists(cubedir):
            self.fail("%s already exists!" % cubedir)
        skeldir = osp.join(BASEDIR, 'skeleton')
643
        longdesc = shortdesc = input(
644
            'Enter a short description for your cube: ')
645
        if verbose:
646
            longdesc = input(
647
                'Enter a long description (leave empty to reuse the short one): ')
648
649
650
651
        dependencies = {
            'six': '>= 1.4.0',
            'cubicweb': '>= %s' % cubicwebversion,
        }
652
        if verbose:
653
            dependencies.update(self._ask_for_dependencies())
Adrien Di Mascio's avatar
Adrien Di Mascio committed
654
655
656
657
        context = {'cubename' : cubename,
                   'distname' : distname,
                   'shortdesc' : shortdesc,
                   'longdesc' : longdesc or shortdesc,
658
                   'dependencies' : dependencies,
Adrien Di Mascio's avatar
Adrien Di Mascio committed
659
                   'version'  : cubicwebversion,
660
                   'year'  : str(date.today().year),
661
662
663
                   'author': self['author'],
                   'author-email': self['author-email'],
                   'author-web-site': self['author-web-site'],
664
665
                   'license': self['license'],
                   'long-license': self.LICENSES[self['license']],
Adrien Di Mascio's avatar
Adrien Di Mascio committed
666
                   }
667
668
669
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
670
                        'cubes.*', 'uiprops.py*')
671
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
672

673
    def _ask_for_dependencies(self):
674
675
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
676
677
678
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
679
680
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
681
                depcubes.append(cube)
682
            if answer == 'type':
683
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
684
                break
685
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
686
                break
687
688
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
689

690
691
692
693

class ExamineLogCommand(Command):
    """Examine a rql log file.

694
    Will print out the following table
695

696
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
697

698
    sorted by descending cumulative time (clock). Time are expressed in seconds.
699

700
    Chances are the lines at the top are the ones that will bring the higher
701
    benefit after optimisation. Start there.
702
    """
703
    arguments = 'rql.log'
704
    name = 'exlog'
705
    options = ()
706

707
708
709
    def run(self, args):
        import re
        requests = {}
710
        for filepath in args:
711
            try:
Rémi Cardona's avatar
Rémi Cardona committed
712
                stream = open(filepath)
713
            except OSError as ex:
714
715
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
716
            for lineno, line in enumerate(stream):
717
718
719
720
721
722
723
724
725
726
727
728
729
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
730
                except Exception as exc:
731
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
732
        stat = []
733
        for rql, times in requests.items():
734
735
736
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
737
738
        stat.sort()
        stat.reverse()
739
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
740
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
741
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
742
743
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
744

745

746
747
748
749
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
750
    min_args = max_args = 1
751
752
    options = [
        ('output-file',
753
         {'type':'string', 'default': None,
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
787
788
789
790
791
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
792
        ]
793
794

    def run(self, args):
795
796
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
797
        from logilab.common.textutils import splitstrip
798
799
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
800
801
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
802
        cubes = splitstrip(args[0])
803
        dev_conf = DevConfiguration(*cubes)
804
805
806
807
808
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
809
810
811
812
813
814
815
816
817
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

834
835
836
837
        if viewer:
            p = Popen((viewer, out))
            p.wait()

838

839
for cmdcls in (UpdateCubicWebCatalogCommand,
840
               UpdateCubeCatalogCommand,
841
842
843
844
845
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)