devctl.py 33.8 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

Adrien Di Mascio's avatar
Adrien Di Mascio committed
23
24
__docformat__ = "restructuredtext en"

25
26
27
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
28
import sys
29
from datetime import datetime, date
30
from os import mkdir, chdir, path as osp
31
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
32

33
34
from six.moves import input

Adrien Di Mascio's avatar
Adrien Di Mascio committed
35
from logilab.common import STD_BLACKLIST
36

Adrien Di Mascio's avatar
Adrien Di Mascio committed
37
from cubicweb.__pkginfo__ import version as cubicwebversion
38
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
39
from cubicweb.cwctl import CWCTL
40
from cubicweb.cwconfig import CubicWebNoAppConfiguration
41
42
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
43
44
45
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

46

47
48
49
50
51
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')


52
53
54
55
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
56
    creating = True
57
    cleanup_unused_appobjects = False
58
59
60
61
62

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
63

64
    def __init__(self, *cubes):
65
66
67
68
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
69
            self.load_site_cubicweb()
70
71
        else:
            self._cubes = ()
72

Adrien Di Mascio's avatar
Adrien Di Mascio committed
73
74
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
75
        return None
76
77
78
79

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
80
81
    def main_config_file(self):
        return None
82
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
83
        pass
84
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
85
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
86
87
    def default_log_file(self):
        return None
88
89
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
90
91
92
93


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
94
    for name, mod in list(sys.modules.items()):
Sylvain Thenault's avatar
Sylvain Thenault committed
95
96
97
98
99
100
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
101
102
103
        if mod.__file__ is None:
            # odd/rare but real
            continue
104
        for path in config.appobjects_path():
Sylvain Thenault's avatar
Sylvain Thenault committed
105
106
107
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
108

Adrien Di Mascio's avatar
Adrien Di Mascio committed
109
110
111
112
113
114
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
115
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
116
    if cubedir:
117
        cube = osp.split(cubedir)[-1]
118
119
120
121
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
122
    else:
123
124
125
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
126
    schema = config.load_schema(remove_unused_rtypes=False)
127
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
128
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
129
130
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
131
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
132
133


134
def _generate_schema_pot(w, vreg, schema, libconfig=None):
135
    from cubicweb.i18n import add_msg
136
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
137
    w('# schema pot file, generated on %s\n'
138
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
139
140
141
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
142
    vregdone = set()
143
    afss = vreg['uicfg']['autoform_section']
144
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
145
    if libconfig is not None:
146
147
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
148
        from cubicweb.cwvreg import CWRegistryStore
149
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
150
        cleanup_sys_modules(libconfig)
151
        libvreg = CWRegistryStore(libconfig)
152
        libvreg.set_schema(libschema) # trigger objects registration
153
        libafss = libvreg['uicfg']['autoform_section']
154
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
155
156
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
157
158
159
160

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
161
    else:
162
        # processing cubicweb itself
163
        libschema = {}
164
165
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
166
167
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
168
    done = set()
169
170
171
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
172
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
173
        etype = eschema.type
174
175
176
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
177
            if not eschema.final:
178
                add_msg(w, 'This %s:' % etype)
179
                add_msg(w, 'New %s' % etype)
180
                add_msg(w, 'add a %s' % etype) # AddNewAction
181
182
183
184
185
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
186
187
188
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
189
        if eschema.final:
190
191
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
192
193
            if rschema.final:
                continue
194
            for tschema in targetschemas:
195

196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
223
224
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
225
    w('# subject and object forms for each relation type\n')
226
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
227
    w('\n')
228
229
230
231
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
232
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
233
        rtype = rschema.type
234
        if rtype not in libschema:
235
236
            # bw compat, necessary until all translation of relation are done
            # properly...
237
            add_msg(w, rtype)
238
            done.add(rtype)
239
240
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
241
            done.add(rschema.description)
242
243
244
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
245
        # add context information only for non-metadata rtypes
246
        if rschema not in NO_I18NCONTEXT:
247
            libsubjects = librschema and librschema.subjects() or ()
248
            for subjschema in rschema.subjects():
249
250
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
251
        if not (rschema.final or rschema.symmetric):
252
            if rschema not in NO_I18NCONTEXT:
253
254
255
256
257
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
258
259
                # bw compat, necessary until all translation of relation are
                # done properly...
260
                add_msg(w, '%s_object' % rtype)
261
        for rdef in rschema.rdefs.values():
262
263
264
265
266
267
268
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
269
    for objid in _iter_vreg_objids(vreg, vregdone):
270
271
272
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
273

274
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
275
    for reg, objdict in vreg.items():
276
277
        if reg in ('boxes', 'contentnavigation'):
            continue
278
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
279
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
280
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
281
                if objid in done:
282
                    break
283
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
284
                if pdefs:
285
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
286
                    done.add(objid)
287
                    break
288

289

Adrien Di Mascio's avatar
Adrien Di Mascio committed
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
307

Adrien Di Mascio's avatar
Adrien Di Mascio committed
308
309
310
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
311
    name = 'i18ncubicweb'
312
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
313
314
315
316

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
317
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
318
319
        import yams
        from logilab.common.fileutils import ensure_fs_mode
320
        from logilab.common.shellutils import globfind, find, rm
321
        from logilab.common.modutils import get_module_files
322
        from cubicweb.i18n import extract_from_tal, execute2
323
        tempdir = tempfile.mkdtemp(prefix='cw-')
324
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
325
326
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
327
328
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
329
330
331
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
332
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
333
334
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
335
        print('TAL', end=' ')
336
337
338
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
339
        print('-> generate .pot files.')
340
341
342
343
344
345
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
346
347
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
348
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
349
                                ]:
350
            potfile = osp.join(tempdir, '%s.pot' % id)
351
352
353
354
355
356
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
357
            if osp.exists(potfile):
358
359
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
360
361
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
362
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
363
364
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
365
        print('-> merging main pot file with existing translations.')
366
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
367
        toedit = []
368
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
369
            target = '%s.po' % lang
370
371
372
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
373
374
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
375
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
376
        # cleanup
377
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
378
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
379
380
381
382
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
383
384


385
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
386
387
388
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
389
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
390
    arguments = '[<cube>...]'
391

Adrien Di Mascio's avatar
Adrien Di Mascio committed
392
393
394
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
395
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
396
        else:
397
398
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
399
400
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
401
402
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
403

404

Adrien Di Mascio's avatar
Adrien Di Mascio committed
405
def update_cubes_catalogs(cubes):
406
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
407
    for cubedir in cubes:
408
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
409
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
410
            continue
411
        try:
412
            toedit = update_cube_catalogs(cubedir)
413
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
414
415
416
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
417
418
419
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
420
            print('*** error while updating catalogs for cube', cubedir)
421
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
422
423
        else:
            # instructions pour la suite
424
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
425
426
427
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
428
429
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
430
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
431

432
433
def update_cube_catalogs(cubedir):
    import shutil
434
    import tempfile
435
436
    from logilab.common.fileutils import ensure_fs_mode
    from logilab.common.shellutils import find, rm
437
    from cubicweb.i18n import extract_from_tal, execute2
438
    cube = osp.basename(osp.normpath(cubedir))
439
    tempdir = tempfile.mkdtemp()
Samuel Trégouët's avatar
Samuel Trégouët committed
440
    print(underline_title('Updating i18n catalogs for cube %s' % cube))
441
    chdir(cubedir)
442
    if osp.exists(osp.join('i18n', 'entities.pot')):
443
        warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'
444
445
446
447
             % osp.join('i18n', 'entities.pot'), DeprecationWarning)
        potfiles = [osp.join('i18n', 'entities.pot')]
    elif osp.exists(osp.join('i18n', 'static-messages.pot')):
        potfiles = [osp.join('i18n', 'static-messages.pot')]
448
449
    else:
        potfiles = []
Samuel Trégouët's avatar
Samuel Trégouët committed
450
451
    print('-> extracting messages:', end=' ')
    print('schema', end=' ')
452
    schemapot = osp.join(tempdir, 'schema.pot')
453
454
455
    potfiles.append(schemapot)
    # explicit close necessary else the file may not be yet flushed when
    # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
456
    schemapotstream = open(schemapot, 'w')
457
458
    generate_schema_pot(schemapotstream.write, cubedir)
    schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
459
    print('TAL', end=' ')
460
    tali18nfile = osp.join(tempdir, 'tali18n.py')
461
    ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST)
462
    extract_from_tal(ptfiles, tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
463
    print('Javascript')
464
465
    jsfiles =  [jsfile for jsfile in find('.', '.js')
                if osp.basename(jsfile).startswith('cub')]
466
    if jsfiles:
467
        tmppotfile = osp.join(tempdir, 'js.pot')
468
469
470
        cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-L', 'java',
               '--from-code=utf-8', '-o', tmppotfile] + jsfiles
        execute2(cmd)
471
        # no pot file created if there are no string to translate
472
        if osp.exists(tmppotfile):
473
            potfiles.append(tmppotfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
474
    print('-> creating cube-specific catalog')
475
    tmppotfile = osp.join(tempdir, 'generated.pot')
476
    cubefiles = find('.', '.py', blacklist=STD_BLACKLIST)
477
    cubefiles.append(tali18nfile)
478
479
480
    cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-o', tmppotfile]
    cmd.extend(cubefiles)
    execute2(cmd)
481
    if osp.exists(tmppotfile): # doesn't exists of no translation string found
482
        potfiles.append(tmppotfile)
483
    potfile = osp.join(tempdir, 'cube.pot')
Samuel Trégouët's avatar
Samuel Trégouët committed
484
    print('-> merging %i .pot files' % len(potfiles))
485
486
487
    cmd = ['msgcat', '-o', potfile]
    cmd.extend(potfiles)
    execute2(cmd)
488
    if not osp.exists(potfile):
Samuel Trégouët's avatar
Samuel Trégouët committed
489
        print('no message catalog for cube', cube, 'nothing to translate')
490
491
492
        # cleanup
        rm(tempdir)
        return ()
Samuel Trégouët's avatar
Samuel Trégouët committed
493
    print('-> merging main pot file with existing translations:', end=' ')
494
    chdir('i18n')
495
    toedit = []
496
    for lang in CubicWebNoAppConfiguration.cw_languages():
Samuel Trégouët's avatar
Samuel Trégouët committed
497
        print(lang, end=' ')
498
        cubepo = '%s.po' % lang
499
        if not osp.exists(cubepo):
500
501
            shutil.copy(potfile, cubepo)
        else:
502
503
            cmd = ['msgmerge','-N','-s','-o', cubepo+'new', cubepo, potfile]
            execute2(cmd)
504
505
            ensure_fs_mode(cubepo)
            shutil.move('%snew' % cubepo, cubepo)
506
        toedit.append(osp.abspath(cubepo))
Samuel Trégouët's avatar
Samuel Trégouët committed
507
    print()
508
509
510
511
512
    # cleanup
    rm(tempdir)
    return toedit


513
514
515
516
517
518
519
# XXX totally broken, fix it
# class LiveServerCommand(Command):
#     """Run a server from within a cube directory.
#     """
#     name = 'live-server'
#     arguments = ''
#     options = ()
520

521
522
523
524
#     def run(self, args):
#         """run the command with its specific arguments"""
#         from cubicweb.devtools.livetest import runserver
#         runserver()
Adrien Di Mascio's avatar
Adrien Di Mascio committed
525
526


527
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
528
529
530
    """Create a new cube.

    <cubename>
531
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
532
533
534
    """
    name = 'newcube'
    arguments = '<cubename>'
535
    min_args = max_args = 1
536
    options = (
537
538
539
540
541
542
543
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
544
545
546
547
548
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
549
550
551
552
553
554
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
573
574
575
576
577
578
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
579
580
        )

581
    LICENSES = {
582
583
584
585
586
587
588
589
590
591
592
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
593
594
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
595
''',
596

597
598
599
600
601
602
603
604
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
605
606
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
607
608
609
610
611
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
612
        }
613

Adrien Di Mascio's avatar
Adrien Di Mascio committed
614
    def run(self, args):
615
        import re
616
        from logilab.common.shellutils import ASK
617
618
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
619
620
            raise BadCommandUsage(
                'cube name must be a valid python module name')
621
        verbose = self.get('verbose')
622
        cubesdir = self.get('directory')
623
624
        if not cubesdir:
            cubespath = ServerConfiguration.cubes_search_path()
625
            if len(cubespath) > 1:
626
627
628
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
629
            cubesdir = cubespath[0]
630
        if not osp.isdir(cubesdir):
Samuel Trégouët's avatar
Samuel Trégouët committed
631
            print("-> creating cubes directory", cubesdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
632
            try:
633
                mkdir(cubesdir)
634
            except OSError as err:
635
636
637
638
                self.fail("failed to create directory %r\n(%s)"
                          % (cubesdir, err))
        cubedir = osp.join(cubesdir, cubename)
        if osp.exists(cubedir):
Dimitri Papadopoulos's avatar
Dimitri Papadopoulos committed
639
            self.fail("%s already exists!" % cubedir)
640
        skeldir = osp.join(BASEDIR, 'skeleton')
641
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
642
        if verbose:
643
            distname = input('Debian name for your cube ? [%s]): '
644
                                 % default_name).strip()
645
            if not distname:
646
                distname = default_name
647
            elif not distname.startswith('cubicweb-'):
648
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
649
650
                    distname = 'cubicweb-' + distname
        else:
651
            distname = default_name
652
        if not re.match('[a-z][-a-z0-9]*$', distname):
653
654
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
655
        longdesc = shortdesc = input(
656
            'Enter a short description for your cube: ')
657
        if verbose:
658
            longdesc = input(
659
                'Enter a long description (leave empty to reuse the short one): ')
660
661
        dependencies = {'cubicweb': '>= %s' % cubicwebversion,
                        'six': '>= 1.4.0',}
662
        if verbose:
663
            dependencies.update(self._ask_for_dependencies())
Adrien Di Mascio's avatar
Adrien Di Mascio committed
664
665
666
667
        context = {'cubename' : cubename,
                   'distname' : distname,
                   'shortdesc' : shortdesc,
                   'longdesc' : longdesc or shortdesc,
668
                   'dependencies' : dependencies,
Adrien Di Mascio's avatar
Adrien Di Mascio committed
669
                   'version'  : cubicwebversion,
670
                   'year'  : str(date.today().year),
671
672
673
                   'author': self['author'],
                   'author-email': self['author-email'],
                   'author-web-site': self['author-web-site'],
674
675
                   'license': self['license'],
                   'long-license': self.LICENSES[self['license']],
Adrien Di Mascio's avatar
Adrien Di Mascio committed
676
                   }
677
678
679
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
680
                        'cubes.*', 'uiprops.py*')
681
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
682

683
    def _ask_for_dependencies(self):
684
685
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
686
687
688
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
689
690
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
691
                depcubes.append(cube)
692
            if answer == 'type':
693
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
694
                break
695
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
696
                break
697
698
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
699

700
701
702
703

class ExamineLogCommand(Command):
    """Examine a rql log file.

704
    Will print out the following table
705

706
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
707

708
    sorted by descending cumulative time (clock). Time are expressed in seconds.
709

710
    Chances are the lines at the top are the ones that will bring the higher
711
    benefit after optimisation. Start there.
712
    """
713
    arguments = 'rql.log'
714
    name = 'exlog'
715
    options = ()
716

717
718
719
    def run(self, args):
        import re
        requests = {}
720
        for filepath in args:
721
            try:
Rémi Cardona's avatar
Rémi Cardona committed
722
                stream = open(filepath)
723
            except OSError as ex:
724
725
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
726
            for lineno, line in enumerate(stream):
727
728
729
730
731
732
733
734
735
736
737
738
739
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
740
                except Exception as exc:
741
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
742
        stat = []
743
        for rql, times in requests.items():
744
745
746
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
747
748
        stat.sort()
        stat.reverse()
749
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
750
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
751
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
752
753
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
754

755

756
757
758
759
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
760
    min_args = max_args = 1
761
762
    options = [
        ('output-file',
763
         {'type':'string', 'default': None,
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
797
798
799
800
801
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
802
        ]
803
804

    def run(self, args):
805
806
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
807
        from logilab.common.textutils import splitstrip
808
809
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
810
811
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
812
        cubes = splitstrip(args[0])
813
        dev_conf = DevConfiguration(*cubes)
814
815
816
817
818
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
819
820
821
822
823
824
825
826
827
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

844
845
846
847
        if viewer:
            p = Popen((viewer, out))
            p.wait()

848

849
for cmdcls in (UpdateCubicWebCatalogCommand,
850
               UpdateCubeCatalogCommand,
851
852
853
854
855
856
               #LiveServerCommand,
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)