devctl.py 33.9 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

Adrien Di Mascio's avatar
Adrien Di Mascio committed
23
24
__docformat__ = "restructuredtext en"

25
26
27
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
28
import sys
29
from datetime import datetime, date
30
from os import mkdir, chdir, path as osp
31
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
32

33
34
from six.moves import input

Adrien Di Mascio's avatar
Adrien Di Mascio committed
35
from logilab.common import STD_BLACKLIST
36

Adrien Di Mascio's avatar
Adrien Di Mascio committed
37
from cubicweb.__pkginfo__ import version as cubicwebversion
38
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
39
from cubicweb.cwctl import CWCTL
40
from cubicweb.cwconfig import CubicWebNoAppConfiguration
41
42
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
43
44
45
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

46

47
48
49
50
51
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')


52
53
54
55
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
56
    creating = True
57
    cleanup_unused_appobjects = False
58
59
60
61
62

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
63

64
    def __init__(self, *cubes):
65
66
67
68
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
69
            self.load_site_cubicweb()
70
71
        else:
            self._cubes = ()
72

Adrien Di Mascio's avatar
Adrien Di Mascio committed
73
74
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
75
        return None
76
77
78
79

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
80
81
    def main_config_file(self):
        return None
82
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
83
        pass
84
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
85
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
86
87
    def default_log_file(self):
        return None
88
89
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
90
91
92
93


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
94
    for name, mod in list(sys.modules.items()):
Sylvain Thenault's avatar
Sylvain Thenault committed
95
96
97
98
99
100
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
101
102
103
        if mod.__file__ is None:
            # odd/rare but real
            continue
104
        for path in config.appobjects_path():
Sylvain Thenault's avatar
Sylvain Thenault committed
105
106
107
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
108

Adrien Di Mascio's avatar
Adrien Di Mascio committed
109
110
111
112
113
114
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
115
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
116
    if cubedir:
117
        cube = osp.split(cubedir)[-1]
118
119
        if cube.startswith('cubicweb_'):
            cube = cube[len('cubicweb_'):]
120
121
122
123
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
124
    else:
125
126
127
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
128
    schema = config.load_schema(remove_unused_rtypes=False)
129
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
130
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
131
132
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
133
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
134
135


136
def _generate_schema_pot(w, vreg, schema, libconfig=None):
137
    from cubicweb.i18n import add_msg
138
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
139
    w('# schema pot file, generated on %s\n'
140
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
141
142
143
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
144
    vregdone = set()
145
    afss = vreg['uicfg']['autoform_section']
146
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
147
    if libconfig is not None:
148
149
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
150
        from cubicweb.cwvreg import CWRegistryStore
151
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
152
        cleanup_sys_modules(libconfig)
153
        libvreg = CWRegistryStore(libconfig)
154
        libvreg.set_schema(libschema) # trigger objects registration
155
        libafss = libvreg['uicfg']['autoform_section']
156
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
157
158
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
159
160
161
162

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
163
    else:
164
        # processing cubicweb itself
165
        libschema = {}
166
167
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
168
169
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
170
    done = set()
171
172
173
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
174
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
175
        etype = eschema.type
176
177
178
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
179
            if not eschema.final:
180
                add_msg(w, 'This %s:' % etype)
181
                add_msg(w, 'New %s' % etype)
182
                add_msg(w, 'add a %s' % etype) # AddNewAction
183
184
185
186
187
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
188
189
190
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
191
        if eschema.final:
192
193
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
194
195
            if rschema.final:
                continue
196
            for tschema in targetschemas:
197

198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
225
226
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
227
    w('# subject and object forms for each relation type\n')
228
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
229
    w('\n')
230
231
232
233
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
234
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
235
        rtype = rschema.type
236
        if rtype not in libschema:
237
238
            # bw compat, necessary until all translation of relation are done
            # properly...
239
            add_msg(w, rtype)
240
            done.add(rtype)
241
242
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
243
            done.add(rschema.description)
244
245
246
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
247
        # add context information only for non-metadata rtypes
248
        if rschema not in NO_I18NCONTEXT:
249
            libsubjects = librschema and librschema.subjects() or ()
250
            for subjschema in rschema.subjects():
251
252
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
253
        if not (rschema.final or rschema.symmetric):
254
            if rschema not in NO_I18NCONTEXT:
255
256
257
258
259
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
260
261
                # bw compat, necessary until all translation of relation are
                # done properly...
262
                add_msg(w, '%s_object' % rtype)
263
        for rdef in rschema.rdefs.values():
264
265
266
267
268
269
270
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
271
    for objid in _iter_vreg_objids(vreg, vregdone):
272
273
274
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
275

276
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
277
    for reg, objdict in vreg.items():
278
279
        if reg in ('boxes', 'contentnavigation'):
            continue
280
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
281
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
282
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
283
                if objid in done:
284
                    break
285
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
286
                if pdefs:
287
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
288
                    done.add(objid)
289
                    break
290

291

Adrien Di Mascio's avatar
Adrien Di Mascio committed
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
309

Adrien Di Mascio's avatar
Adrien Di Mascio committed
310
311
312
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
313
    name = 'i18ncubicweb'
314
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
315
316
317
318

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
319
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
320
321
        import yams
        from logilab.common.fileutils import ensure_fs_mode
322
        from logilab.common.shellutils import globfind, find, rm
323
        from logilab.common.modutils import get_module_files
324
        from cubicweb.i18n import extract_from_tal, execute2
325
        tempdir = tempfile.mkdtemp(prefix='cw-')
326
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
327
328
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
329
330
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
331
332
333
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
334
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
335
336
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
337
        print('TAL', end=' ')
338
339
340
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
341
        print('-> generate .pot files.')
342
343
344
345
346
347
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
348
349
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
350
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
351
                                ]:
352
            potfile = osp.join(tempdir, '%s.pot' % id)
353
354
355
356
357
358
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
359
            if osp.exists(potfile):
360
361
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
362
363
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
364
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
365
366
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
367
        print('-> merging main pot file with existing translations.')
368
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
369
        toedit = []
370
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
371
            target = '%s.po' % lang
372
373
374
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
375
376
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
377
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
378
        # cleanup
379
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
380
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
381
382
383
384
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
385
386


387
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
388
389
390
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
391
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
392
    arguments = '[<cube>...]'
393

Adrien Di Mascio's avatar
Adrien Di Mascio committed
394
395
396
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
397
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
398
        else:
399
400
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
401
402
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
403
404
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
405

406

Adrien Di Mascio's avatar
Adrien Di Mascio committed
407
def update_cubes_catalogs(cubes):
408
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
409
    for cubedir in cubes:
410
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
411
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
412
            continue
413
        try:
414
            toedit = update_cube_catalogs(cubedir)
415
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
416
417
418
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
419
420
421
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
422
            print('*** error while updating catalogs for cube', cubedir)
423
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
424
425
        else:
            # instructions pour la suite
426
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
427
428
429
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
430
431
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
432
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
433

434
435
def update_cube_catalogs(cubedir):
    import shutil
436
    import tempfile
437
438
    from logilab.common.fileutils import ensure_fs_mode
    from logilab.common.shellutils import find, rm
439
    from cubicweb.i18n import extract_from_tal, execute2
440
    cube = osp.basename(osp.normpath(cubedir))
441
    tempdir = tempfile.mkdtemp()
Samuel Trégouët's avatar
Samuel Trégouët committed
442
    print(underline_title('Updating i18n catalogs for cube %s' % cube))
443
    chdir(cubedir)
444
    if osp.exists(osp.join('i18n', 'entities.pot')):
445
        warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'
446
447
448
449
             % osp.join('i18n', 'entities.pot'), DeprecationWarning)
        potfiles = [osp.join('i18n', 'entities.pot')]
    elif osp.exists(osp.join('i18n', 'static-messages.pot')):
        potfiles = [osp.join('i18n', 'static-messages.pot')]
450
451
    else:
        potfiles = []
Samuel Trégouët's avatar
Samuel Trégouët committed
452
453
    print('-> extracting messages:', end=' ')
    print('schema', end=' ')
454
    schemapot = osp.join(tempdir, 'schema.pot')
455
456
457
    potfiles.append(schemapot)
    # explicit close necessary else the file may not be yet flushed when
    # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
458
    schemapotstream = open(schemapot, 'w')
459
460
    generate_schema_pot(schemapotstream.write, cubedir)
    schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
461
    print('TAL', end=' ')
462
    tali18nfile = osp.join(tempdir, 'tali18n.py')
463
    ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST)
464
    extract_from_tal(ptfiles, tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
465
    print('Javascript')
466
467
    jsfiles =  [jsfile for jsfile in find('.', '.js')
                if osp.basename(jsfile).startswith('cub')]
468
    if jsfiles:
469
        tmppotfile = osp.join(tempdir, 'js.pot')
470
471
472
        cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-L', 'java',
               '--from-code=utf-8', '-o', tmppotfile] + jsfiles
        execute2(cmd)
473
        # no pot file created if there are no string to translate
474
        if osp.exists(tmppotfile):
475
            potfiles.append(tmppotfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
476
    print('-> creating cube-specific catalog')
477
    tmppotfile = osp.join(tempdir, 'generated.pot')
478
    cubefiles = find('.', '.py', blacklist=STD_BLACKLIST)
479
    cubefiles.append(tali18nfile)
480
481
482
    cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-o', tmppotfile]
    cmd.extend(cubefiles)
    execute2(cmd)
483
    if osp.exists(tmppotfile): # doesn't exists of no translation string found
484
        potfiles.append(tmppotfile)
485
    potfile = osp.join(tempdir, 'cube.pot')
Samuel Trégouët's avatar
Samuel Trégouët committed
486
    print('-> merging %i .pot files' % len(potfiles))
487
488
489
    cmd = ['msgcat', '-o', potfile]
    cmd.extend(potfiles)
    execute2(cmd)
490
    if not osp.exists(potfile):
Samuel Trégouët's avatar
Samuel Trégouët committed
491
        print('no message catalog for cube', cube, 'nothing to translate')
492
493
494
        # cleanup
        rm(tempdir)
        return ()
Samuel Trégouët's avatar
Samuel Trégouët committed
495
    print('-> merging main pot file with existing translations:', end=' ')
496
    chdir('i18n')
497
    toedit = []
498
    for lang in CubicWebNoAppConfiguration.cw_languages():
Samuel Trégouët's avatar
Samuel Trégouët committed
499
        print(lang, end=' ')
500
        cubepo = '%s.po' % lang
501
        if not osp.exists(cubepo):
502
503
            shutil.copy(potfile, cubepo)
        else:
504
505
            cmd = ['msgmerge','-N','-s','-o', cubepo+'new', cubepo, potfile]
            execute2(cmd)
506
507
            ensure_fs_mode(cubepo)
            shutil.move('%snew' % cubepo, cubepo)
508
        toedit.append(osp.abspath(cubepo))
Samuel Trégouët's avatar
Samuel Trégouët committed
509
    print()
510
511
512
513
514
    # cleanup
    rm(tempdir)
    return toedit


515
516
517
518
519
520
521
# XXX totally broken, fix it
# class LiveServerCommand(Command):
#     """Run a server from within a cube directory.
#     """
#     name = 'live-server'
#     arguments = ''
#     options = ()
522

523
524
525
526
#     def run(self, args):
#         """run the command with its specific arguments"""
#         from cubicweb.devtools.livetest import runserver
#         runserver()
Adrien Di Mascio's avatar
Adrien Di Mascio committed
527
528


529
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
530
531
532
    """Create a new cube.

    <cubename>
533
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
534
535
536
    """
    name = 'newcube'
    arguments = '<cubename>'
537
    min_args = max_args = 1
538
    options = (
539
540
541
542
543
544
545
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
546
547
548
549
550
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
551
552
553
554
555
556
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
575
576
577
578
579
580
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
581
582
        )

583
    LICENSES = {
584
585
586
587
588
589
590
591
592
593
594
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
595
596
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
597
''',
598

599
600
601
602
603
604
605
606
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
607
608
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
609
610
611
612
613
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
614
        }
615

Adrien Di Mascio's avatar
Adrien Di Mascio committed
616
    def run(self, args):
617
        import re
618
        from logilab.common.shellutils import ASK
619
620
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
621
622
            raise BadCommandUsage(
                'cube name must be a valid python module name')
623
        verbose = self.get('verbose')
624
625
        destdir = self.get('directory')
        if not destdir:
626
            cubespath = ServerConfiguration.cubes_search_path()
627
            if len(cubespath) > 1:
628
629
630
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
631
632
633
            destdir = cubespath[0]
        if not osp.isdir(destdir):
            print("-> creating cubes directory", destdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
634
            try:
635
                mkdir(destdir)
636
            except OSError as err:
637
                self.fail("failed to create directory %r\n(%s)"
638
                          % (destdir, err))
639
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
640
        if verbose:
641
            distname = input('Debian name for your cube ? [%s]): '
642
                             % default_name).strip()
643
            if not distname:
644
                distname = default_name
645
            elif not distname.startswith('cubicweb-'):
646
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
647
648
                    distname = 'cubicweb-' + distname
        else:
649
            distname = default_name
650
        if not re.match('[a-z][-a-z0-9]*$', distname):
651
652
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
653
654
655
656
        cubedir = osp.join(destdir, distname)
        if osp.exists(cubedir):
            self.fail("%s already exists!" % cubedir)
        skeldir = osp.join(BASEDIR, 'skeleton')
657
        longdesc = shortdesc = input(
658
            'Enter a short description for your cube: ')
659
        if verbose:
660
            longdesc = input(
661
                'Enter a long description (leave empty to reuse the short one): ')
662
663
664
665
        dependencies = {
            'six': '>= 1.4.0',
            'cubicweb': '>= %s' % cubicwebversion,
        }
666
        if verbose:
667
            dependencies.update(self._ask_for_dependencies())
Adrien Di Mascio's avatar
Adrien Di Mascio committed
668
669
670
671
        context = {'cubename' : cubename,
                   'distname' : distname,
                   'shortdesc' : shortdesc,
                   'longdesc' : longdesc or shortdesc,
672
                   'dependencies' : dependencies,
Adrien Di Mascio's avatar
Adrien Di Mascio committed
673
                   'version'  : cubicwebversion,
674
                   'year'  : str(date.today().year),
675
676
677
                   'author': self['author'],
                   'author-email': self['author-email'],
                   'author-web-site': self['author-web-site'],
678
679
                   'license': self['license'],
                   'long-license': self.LICENSES[self['license']],
Adrien Di Mascio's avatar
Adrien Di Mascio committed
680
                   }
681
682
683
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
684
                        'cubes.*', 'uiprops.py*')
685
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
686

687
    def _ask_for_dependencies(self):
688
689
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
690
691
692
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
693
694
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
695
                depcubes.append(cube)
696
            if answer == 'type':
697
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
698
                break
699
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
700
                break
701
702
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
703

704
705
706
707

class ExamineLogCommand(Command):
    """Examine a rql log file.

708
    Will print out the following table
709

710
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
711

712
    sorted by descending cumulative time (clock). Time are expressed in seconds.
713

714
    Chances are the lines at the top are the ones that will bring the higher
715
    benefit after optimisation. Start there.
716
    """
717
    arguments = 'rql.log'
718
    name = 'exlog'
719
    options = ()
720

721
722
723
    def run(self, args):
        import re
        requests = {}
724
        for filepath in args:
725
            try:
Rémi Cardona's avatar
Rémi Cardona committed
726
                stream = open(filepath)
727
            except OSError as ex:
728
729
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
730
            for lineno, line in enumerate(stream):
731
732
733
734
735
736
737
738
739
740
741
742
743
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
744
                except Exception as exc:
745
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
746
        stat = []
747
        for rql, times in requests.items():
748
749
750
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
751
752
        stat.sort()
        stat.reverse()
753
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
754
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
755
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
756
757
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
758

759

760
761
762
763
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
764
    min_args = max_args = 1
765
766
    options = [
        ('output-file',
767
         {'type':'string', 'default': None,
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
801
802
803
804
805
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
806
        ]
807
808

    def run(self, args):
809
810
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
811
        from logilab.common.textutils import splitstrip
812
813
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
814
815
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
816
        cubes = splitstrip(args[0])
817
        dev_conf = DevConfiguration(*cubes)
818
819
820
821
822
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
823
824
825
826
827
828
829
830
831
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

848
849
850
851
        if viewer:
            p = Popen((viewer, out))
            p.wait()

852

853
for cmdcls in (UpdateCubicWebCatalogCommand,
854
               UpdateCubeCatalogCommand,
855
856
857
858
859
860
               #LiveServerCommand,
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)