devctl.py 33.7 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
3
4
5
6
7
8
9
10
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
17
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
Samuel Trégouët's avatar
Samuel Trégouët committed
21
from __future__ import print_function
22

Adrien Di Mascio's avatar
Adrien Di Mascio committed
23
24
__docformat__ = "restructuredtext en"

25
26
27
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
28
import sys
29
from datetime import datetime, date
30
from os import mkdir, chdir, path as osp
31
from warnings import warn
Adrien Di Mascio's avatar
Adrien Di Mascio committed
32

33
34
from six.moves import input

Adrien Di Mascio's avatar
Adrien Di Mascio committed
35
from logilab.common import STD_BLACKLIST
36

Adrien Di Mascio's avatar
Adrien Di Mascio committed
37
from cubicweb.__pkginfo__ import version as cubicwebversion
38
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
39
from cubicweb.cwctl import CWCTL
40
from cubicweb.cwconfig import CubicWebNoAppConfiguration
41
42
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
                                 underline_title)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
43
44
45
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

46

47
48
49
50
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
Adrien Di Mascio's avatar
Adrien Di Mascio committed
51
    creating = True
52
    cleanup_unused_appobjects = False
53
54
55
56
57

    cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path
                               | WebConfiguration.cubicweb_appobject_path)
    cube_appobject_path = (ServerConfiguration.cube_appobject_path
                           | WebConfiguration.cube_appobject_path)
Sylvain Thenault's avatar
Sylvain Thenault committed
58

59
    def __init__(self, *cubes):
60
61
62
63
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
                self.expand_cubes(cubes, with_recommends=True))
64
            self.load_site_cubicweb()
65
66
        else:
            self._cubes = ()
67

Adrien Di Mascio's avatar
Adrien Di Mascio committed
68
69
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
70
        return None
71
72
73
74

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
75
76
    def main_config_file(self):
        return None
77
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
78
        pass
79
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
80
        pass
Sylvain Thenault's avatar
Sylvain Thenault committed
81
82
    def default_log_file(self):
        return None
83
84
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
85
86
87
88


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
89
    for name, mod in list(sys.modules.items()):
Sylvain Thenault's avatar
Sylvain Thenault committed
90
91
92
93
94
95
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
96
97
98
        if mod.__file__ is None:
            # odd/rare but real
            continue
99
        for path in config.appobjects_path():
Sylvain Thenault's avatar
Sylvain Thenault committed
100
101
102
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
103

Adrien Di Mascio's avatar
Adrien Di Mascio committed
104
105
106
107
108
109
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
110
    from cubicweb.cwvreg import CWRegistryStore
Adrien Di Mascio's avatar
Adrien Di Mascio committed
111
    if cubedir:
112
        cube = osp.split(cubedir)[-1]
113
114
115
116
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
117
    else:
118
119
120
        config = DevConfiguration()
        cube = libconfig = None
    cleanup_sys_modules(config)
121
    schema = config.load_schema(remove_unused_rtypes=False)
122
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
123
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
124
125
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
126
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
127
128


129
def _generate_schema_pot(w, vreg, schema, libconfig=None):
130
    from cubicweb.i18n import add_msg
131
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
132
    w('# schema pot file, generated on %s\n'
133
      % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
134
135
136
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
137
    vregdone = set()
138
    afss = vreg['uicfg']['autoform_section']
139
    aiams = vreg['uicfg']['actionbox_appearsin_addmenu']
140
    if libconfig is not None:
141
142
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
143
        from cubicweb.cwvreg import CWRegistryStore
144
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
145
        cleanup_sys_modules(libconfig)
146
        libvreg = CWRegistryStore(libconfig)
147
        libvreg.set_schema(libschema) # trigger objects registration
148
        libafss = libvreg['uicfg']['autoform_section']
149
        libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']
150
151
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
152
153
154
155

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
            return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))
                       for rtag in rtags)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
156
    else:
157
        # processing cubicweb itself
158
        libschema = {}
159
160
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
161
162
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
163
    done = set()
164
165
166
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
167
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
168
        etype = eschema.type
169
170
171
        if etype not in libschema:
            add_msg(w, etype)
            add_msg(w, '%s_plural' % etype)
172
            if not eschema.final:
173
                add_msg(w, 'This %s:' % etype)
174
                add_msg(w, 'New %s' % etype)
175
                add_msg(w, 'add a %s' % etype) # AddNewAction
176
177
178
179
180
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
                    add_msg(w, 'This %s' % etype)
181
182
183
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
184
        if eschema.final:
185
186
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
187
188
            if rschema.final:
                continue
189
            for tschema in targetschemas:
190

191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
                    if 'main_inlined' in fsections and not \
                            is_in_lib(libafss, eschema, rschema, role, tschema,
                                      lambda x: 'main_inlined' in x):
                        add_msg(w, 'add a %s' % tschema,
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        add_msg(w, str(tschema),
                                'inlined:%s.%s.%s' % (etype, rschema, role))
                        break

                for aiam in aiams:
                    if aiam.etype_get(eschema, rschema, role, tschema) and not \
                            is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema,
                                                         tschema, role)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
                                tschema, eschema, rschema, tschema)
                        else:
                            label = 'add %s %s %s %s' % (tschema, rschema,
                                                         eschema, role)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
                                tschema, tschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
218
219
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
Adrien Di Mascio's avatar
Adrien Di Mascio committed
220
    w('# subject and object forms for each relation type\n')
221
    w('# (no object form for final or symmetric relation types)\n')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
222
    w('\n')
223
224
225
226
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
227
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
228
        rtype = rschema.type
229
        if rtype not in libschema:
230
231
            # bw compat, necessary until all translation of relation are done
            # properly...
232
            add_msg(w, rtype)
233
            done.add(rtype)
234
235
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
236
            done.add(rschema.description)
237
238
239
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
240
        # add context information only for non-metadata rtypes
241
        if rschema not in NO_I18NCONTEXT:
242
            libsubjects = librschema and librschema.subjects() or ()
243
            for subjschema in rschema.subjects():
244
245
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
246
        if not (rschema.final or rschema.symmetric):
247
            if rschema not in NO_I18NCONTEXT:
248
249
250
251
252
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
                        add_msg(w, '%s_object' % rtype, objschema.type)
            if rtype not in libschema:
253
254
                # bw compat, necessary until all translation of relation are
                # done properly...
255
                add_msg(w, '%s_object' % rtype)
256
        for rdef in rschema.rdefs.values():
257
258
259
260
261
262
263
            if not rdef.description or rdef.description in done:
                continue
            if (librschema is None or
                (rdef.subject, rdef.object) not in librschema.rdefs or
                librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):
                add_msg(w, rdef.description)
            done.add(rdef.description)
264
    for objid in _iter_vreg_objids(vreg, vregdone):
265
266
267
        add_msg(w, '%s_description' % objid)
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
268

269
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
270
    for reg, objdict in vreg.items():
271
272
        if reg in ('boxes', 'contentnavigation'):
            continue
273
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
274
            for obj in objects:
Sylvain Thénault's avatar
Sylvain Thénault committed
275
                objid = '%s_%s' % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
276
                if objid in done:
277
                    break
278
                pdefs = getattr(obj, 'cw_property_defs', {})
Sylvain Thénault's avatar
Sylvain Thénault committed
279
                if pdefs:
280
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
281
                    done.add(objid)
282
                    break
283

284

Adrien Di Mascio's avatar
Adrien Di Mascio committed
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
302

Adrien Di Mascio's avatar
Adrien Di Mascio committed
303
304
305
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
306
    name = 'i18ncubicweb'
307
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
308
309
310
311

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
312
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
313
314
        import yams
        from logilab.common.fileutils import ensure_fs_mode
315
        from logilab.common.shellutils import globfind, find, rm
316
        from logilab.common.modutils import get_module_files
317
        from cubicweb.i18n import extract_from_tal, execute2
318
        tempdir = tempfile.mkdtemp(prefix='cw-')
319
        cwi18ndir = WebConfiguration.i18n_lib_dir()
Samuel Trégouët's avatar
Samuel Trégouët committed
320
321
        print('-> extract messages:', end=' ')
        print('schema', end=' ')
322
323
        schemapot = osp.join(tempdir, 'schema.pot')
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
324
325
326
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
327
        schemapotstream = open(schemapot, 'w')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
328
329
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
330
        print('TAL', end=' ')
331
332
333
        tali18nfile = osp.join(tempdir, 'tali18n.py')
        extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
                         tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
334
        print('-> generate .pot files.')
335
336
337
338
339
340
        pyfiles = get_module_files(BASEDIR)
        pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
        schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
        jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')
        for id, files, lang in [('pycubicweb', pyfiles, None),
                                ('schemadescr', schemafiles, None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
341
342
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
343
                                ('js', jsfiles, 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
344
                                ]:
345
            potfile = osp.join(tempdir, '%s.pot' % id)
346
347
348
349
350
351
            cmd = ['xgettext', '--no-location', '--omit-header', '-k_']
            if lang is not None:
                cmd.extend(['-L', lang])
            cmd.extend(['-o', potfile])
            cmd.extend(files)
            execute2(cmd)
352
            if osp.exists(potfile):
353
354
                potfiles.append(potfile)
            else:
Samuel Trégouët's avatar
Samuel Trégouët committed
355
356
                print('-> WARNING: %s file was not generated' % potfile)
        print('-> merging %i .pot files' % len(potfiles))
357
        cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
358
359
        cmd = ['msgcat', '-o', cubicwebpot] + potfiles
        execute2(cmd)
Samuel Trégouët's avatar
Samuel Trégouët committed
360
        print('-> merging main pot file with existing translations.')
361
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
362
        toedit = []
363
        for lang in CubicWebNoAppConfiguration.cw_languages():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
364
            target = '%s.po' % lang
365
366
367
            cmd = ['msgmerge', '-N', '--sort-output', '-o',
                   target+'new', target, cubicwebpot]
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
368
369
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
370
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
371
        # cleanup
372
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
373
        # instructions pour la suite
Samuel Trégouët's avatar
Samuel Trégouët committed
374
375
376
377
        print('-> regenerated CubicWeb\'s .po catalogs.')
        print('\nYou can now edit the following files:')
        print('* ' + '\n* '.join(toedit))
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
378
379


380
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
381
382
383
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
384
    name = 'i18ncube'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
385
    arguments = '[<cube>...]'
386

Adrien Di Mascio's avatar
Adrien Di Mascio committed
387
388
389
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
390
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
391
        else:
392
393
            cubes = [DevConfiguration.cube_dir(cube)
                     for cube in DevConfiguration.available_cubes()]
394
395
            cubes = [cubepath for cubepath in cubes
                     if osp.exists(osp.join(cubepath, 'i18n'))]
396
397
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
398

399

Adrien Di Mascio's avatar
Adrien Di Mascio committed
400
def update_cubes_catalogs(cubes):
401
    from subprocess import CalledProcessError
Adrien Di Mascio's avatar
Adrien Di Mascio committed
402
    for cubedir in cubes:
403
        if not osp.isdir(cubedir):
Samuel Trégouët's avatar
Samuel Trégouët committed
404
            print('-> ignoring %s that is not a directory.' % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
405
            continue
406
        try:
407
            toedit = update_cube_catalogs(cubedir)
408
        except CalledProcessError as exc:
Samuel Trégouët's avatar
Samuel Trégouët committed
409
410
411
            print('\n*** error while updating catalogs for cube', cubedir)
            print('cmd:\n%s' % exc.cmd)
            print('stdout:\n%s\nstderr:\n%s' % exc.data)
412
413
414
        except Exception:
            import traceback
            traceback.print_exc()
Samuel Trégouët's avatar
Samuel Trégouët committed
415
            print('*** error while updating catalogs for cube', cubedir)
416
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
417
418
        else:
            # instructions pour la suite
419
            if toedit:
Samuel Trégouët's avatar
Samuel Trégouët committed
420
421
422
                print('-> regenerated .po catalogs for cube %s.' % cubedir)
                print('\nYou can now edit the following files:')
                print('* ' + '\n* '.join(toedit))
423
424
                print ('When you are done, run "cubicweb-ctl i18ninstance '
                       '<yourinstance>" to see changes in your instances.')
425
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
426

427
428
def update_cube_catalogs(cubedir):
    import shutil
429
    import tempfile
430
431
    from logilab.common.fileutils import ensure_fs_mode
    from logilab.common.shellutils import find, rm
432
    from cubicweb.i18n import extract_from_tal, execute2
433
    cube = osp.basename(osp.normpath(cubedir))
434
    tempdir = tempfile.mkdtemp()
Samuel Trégouët's avatar
Samuel Trégouët committed
435
    print(underline_title('Updating i18n catalogs for cube %s' % cube))
436
    chdir(cubedir)
437
    if osp.exists(osp.join('i18n', 'entities.pot')):
438
        warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'
439
440
441
442
             % osp.join('i18n', 'entities.pot'), DeprecationWarning)
        potfiles = [osp.join('i18n', 'entities.pot')]
    elif osp.exists(osp.join('i18n', 'static-messages.pot')):
        potfiles = [osp.join('i18n', 'static-messages.pot')]
443
444
    else:
        potfiles = []
Samuel Trégouët's avatar
Samuel Trégouët committed
445
446
    print('-> extracting messages:', end=' ')
    print('schema', end=' ')
447
    schemapot = osp.join(tempdir, 'schema.pot')
448
449
450
    potfiles.append(schemapot)
    # explicit close necessary else the file may not be yet flushed when
    # we'll using it below
Rémi Cardona's avatar
Rémi Cardona committed
451
    schemapotstream = open(schemapot, 'w')
452
453
    generate_schema_pot(schemapotstream.write, cubedir)
    schemapotstream.close()
Samuel Trégouët's avatar
Samuel Trégouët committed
454
    print('TAL', end=' ')
455
456
457
    tali18nfile = osp.join(tempdir, 'tali18n.py')
    ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST+('test',))
    extract_from_tal(ptfiles, tali18nfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
458
    print('Javascript')
459
460
    jsfiles =  [jsfile for jsfile in find('.', '.js')
                if osp.basename(jsfile).startswith('cub')]
461
    if jsfiles:
462
        tmppotfile = osp.join(tempdir, 'js.pot')
463
464
465
        cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-L', 'java',
               '--from-code=utf-8', '-o', tmppotfile] + jsfiles
        execute2(cmd)
466
        # no pot file created if there are no string to translate
467
        if osp.exists(tmppotfile):
468
            potfiles.append(tmppotfile)
Samuel Trégouët's avatar
Samuel Trégouët committed
469
    print('-> creating cube-specific catalog')
470
    tmppotfile = osp.join(tempdir, 'generated.pot')
471
472
    cubefiles = find('.', '.py', blacklist=STD_BLACKLIST+('test',))
    cubefiles.append(tali18nfile)
473
474
475
    cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-o', tmppotfile]
    cmd.extend(cubefiles)
    execute2(cmd)
476
    if osp.exists(tmppotfile): # doesn't exists of no translation string found
477
        potfiles.append(tmppotfile)
478
    potfile = osp.join(tempdir, 'cube.pot')
Samuel Trégouët's avatar
Samuel Trégouët committed
479
    print('-> merging %i .pot files' % len(potfiles))
480
481
482
    cmd = ['msgcat', '-o', potfile]
    cmd.extend(potfiles)
    execute2(cmd)
483
    if not osp.exists(potfile):
Samuel Trégouët's avatar
Samuel Trégouët committed
484
        print('no message catalog for cube', cube, 'nothing to translate')
485
486
487
        # cleanup
        rm(tempdir)
        return ()
Samuel Trégouët's avatar
Samuel Trégouët committed
488
    print('-> merging main pot file with existing translations:', end=' ')
489
    chdir('i18n')
490
    toedit = []
491
    for lang in CubicWebNoAppConfiguration.cw_languages():
Samuel Trégouët's avatar
Samuel Trégouët committed
492
        print(lang, end=' ')
493
        cubepo = '%s.po' % lang
494
        if not osp.exists(cubepo):
495
496
            shutil.copy(potfile, cubepo)
        else:
497
498
            cmd = ['msgmerge','-N','-s','-o', cubepo+'new', cubepo, potfile]
            execute2(cmd)
499
500
            ensure_fs_mode(cubepo)
            shutil.move('%snew' % cubepo, cubepo)
501
        toedit.append(osp.abspath(cubepo))
Samuel Trégouët's avatar
Samuel Trégouët committed
502
    print()
503
504
505
506
507
    # cleanup
    rm(tempdir)
    return toedit


508
509
510
511
512
513
514
# XXX totally broken, fix it
# class LiveServerCommand(Command):
#     """Run a server from within a cube directory.
#     """
#     name = 'live-server'
#     arguments = ''
#     options = ()
515

516
517
518
519
#     def run(self, args):
#         """run the command with its specific arguments"""
#         from cubicweb.devtools.livetest import runserver
#         runserver()
Adrien Di Mascio's avatar
Adrien Di Mascio committed
520
521


522
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
523
524
525
    """Create a new cube.

    <cubename>
526
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
527
528
529
    """
    name = 'newcube'
    arguments = '<cubename>'
530
    min_args = max_args = 1
531
    options = (
532
533
534
535
536
537
538
        ("layout",
         {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',
          'default': 'simple', 'choices': ('simple', 'full'),
          'help': 'cube layout. You\'ll get a minimal cube with the "simple" \
layout, and a full featured cube with "full" layout.',
          }
         ),
539
540
541
542
543
        ("directory",
         {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',
          'help': 'directory where the new cube should be created',
          }
         ),
544
545
546
547
548
549
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
568
569
570
571
572
573
        ("license",
         {'short': 'l', 'type' : 'choice', 'metavar': '<license>',
          'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),
          'help': 'cube license',
          }
         ),
574
575
        )

576
    LICENSES = {
577
578
579
580
581
582
583
584
585
586
587
        'LGPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
588
589
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
590
''',
591

592
593
594
595
596
597
598
599
        'GPL': '''\
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
600
601
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
602
603
604
605
606
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
''',
        '': '# INSERT LICENSE HERE'
607
        }
608

Adrien Di Mascio's avatar
Adrien Di Mascio committed
609
    def run(self, args):
610
        import re
611
        from logilab.common.shellutils import ASK
612
613
        cubename = args[0]
        if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):
614
615
            raise BadCommandUsage(
                'cube name must be a valid python module name')
616
        verbose = self.get('verbose')
617
        cubesdir = self.get('directory')
618
619
        if not cubesdir:
            cubespath = ServerConfiguration.cubes_search_path()
620
            if len(cubespath) > 1:
621
622
623
                raise BadCommandUsage(
                    "can't guess directory where to put the new cube."
                    " Please specify it using the --directory option")
624
            cubesdir = cubespath[0]
625
        if not osp.isdir(cubesdir):
Samuel Trégouët's avatar
Samuel Trégouët committed
626
            print("-> creating cubes directory", cubesdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
627
            try:
628
                mkdir(cubesdir)
629
            except OSError as err:
630
631
632
633
                self.fail("failed to create directory %r\n(%s)"
                          % (cubesdir, err))
        cubedir = osp.join(cubesdir, cubename)
        if osp.exists(cubedir):
Dimitri Papadopoulos's avatar
Dimitri Papadopoulos committed
634
            self.fail("%s already exists!" % cubedir)
635
        skeldir = osp.join(BASEDIR, 'skeleton')
636
        default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
637
        if verbose:
638
            distname = input('Debian name for your cube ? [%s]): '
639
                                 % default_name).strip()
640
            if not distname:
641
                distname = default_name
642
            elif not distname.startswith('cubicweb-'):
643
                if ASK.confirm('Do you mean cubicweb-%s ?' % distname):
644
645
                    distname = 'cubicweb-' + distname
        else:
646
            distname = default_name
647
        if not re.match('[a-z][-a-z0-9]*$', distname):
648
649
            raise BadCommandUsage(
                'cube distname should be a valid debian package name')
650
        longdesc = shortdesc = input(
651
            'Enter a short description for your cube: ')
652
        if verbose:
653
            longdesc = input(
654
                'Enter a long description (leave empty to reuse the short one): ')
655
656
        dependencies = {'cubicweb': '>= %s' % cubicwebversion,
                        'six': '>= 1.4.0',}
657
        if verbose:
658
            dependencies.update(self._ask_for_dependencies())
Adrien Di Mascio's avatar
Adrien Di Mascio committed
659
660
661
662
        context = {'cubename' : cubename,
                   'distname' : distname,
                   'shortdesc' : shortdesc,
                   'longdesc' : longdesc or shortdesc,
663
                   'dependencies' : dependencies,
Adrien Di Mascio's avatar
Adrien Di Mascio committed
664
                   'version'  : cubicwebversion,
665
                   'year'  : str(date.today().year),
666
667
668
                   'author': self['author'],
                   'author-email': self['author-email'],
                   'author-web-site': self['author-web-site'],
669
670
                   'license': self['license'],
                   'long-license': self.LICENSES[self['license']],
Adrien Di Mascio's avatar
Adrien Di Mascio committed
671
                   }
672
673
674
        exclude = SKEL_EXCLUDE
        if self['layout'] == 'simple':
            exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',
675
                        'cubes.*', 'uiprops.py*')
676
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
677

678
    def _ask_for_dependencies(self):
679
680
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
681
682
683
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
            answer = ASK.ask("Depends on cube %s? " % cube,
684
685
                             ('N','y','skip','type'), 'N')
            if answer == 'y':
686
                depcubes.append(cube)
687
            if answer == 'type':
688
                depcubes = splitstrip(input('type dependencies: '))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
689
                break
690
            elif answer == 'skip':
Adrien Di Mascio's avatar
Adrien Di Mascio committed
691
                break
692
693
        return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))
                    for cube in depcubes)
694

695
696
697
698

class ExamineLogCommand(Command):
    """Examine a rql log file.

699
    Will print out the following table
700

701
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
702

703
    sorted by descending cumulative time (clock). Time are expressed in seconds.
704

705
    Chances are the lines at the top are the ones that will bring the higher
706
    benefit after optimisation. Start there.
707
    """
708
    arguments = 'rql.log'
709
    name = 'exlog'
710
    options = ()
711

712
713
714
    def run(self, args):
        import re
        requests = {}
715
        for filepath in args:
716
            try:
Rémi Cardona's avatar
Rémi Cardona committed
717
                stream = open(filepath)
718
            except OSError as ex:
719
720
                raise BadCommandUsage("can't open rql log file %s: %s"
                                      % (filepath, ex))
721
            for lineno, line in enumerate(stream):
722
723
724
725
726
727
728
729
730
731
732
733
734
                if not ' WHERE ' in line:
                    continue
                try:
                    rql, time = line.split('--')
                    rql = re.sub("(\'\w+': \d*)", '', rql)
                    if '{' in rql:
                        rql = rql[:rql.index('{')]
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
                    req.append( (clocktime, cputime) )
735
                except Exception as exc:
736
                    sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
737
        stat = []
738
        for rql, times in requests.items():
739
740
741
            stat.append( (sum(time[0] for time in times),
                          sum(time[1] for time in times),
                          len(times), rql) )
742
743
        stat.sort()
        stat.reverse()
744
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
Samuel Trégouët's avatar
Samuel Trégouët committed
745
        print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
746
        for clocktime, cputime, occ, rql in stat:
Samuel Trégouët's avatar
Samuel Trégouët committed
747
748
            print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
                                            cputime, occ, rql))
749

750

751
752
753
754
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
    name = "schema"
    arguments = '<cube>'
755
    min_args = max_args = 1
756
757
    options = [
        ('output-file',
758
         {'type':'string', 'default': None,
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
          'metavar': '<file>', 'short':'o', 'help':'output image file',
          'input':False,
          }),
        ('viewer',
         {'type': 'string', 'default':None,
          'short': "d", 'metavar':'<cmd>',
          'help':'command use to view the generated file (empty for none)',
          }),
        ('show-meta',
         {'action': 'store_true', 'default':False,
          'short': "m", 'metavar': "<yN>",
          'help':'include meta and internal entities in schema',
          }),
        ('show-workflow',
         {'action': 'store_true', 'default':False,
          'short': "w", 'metavar': "<yN>",
          'help':'include workflow entities in schema',
          }),
        ('show-cw-user',
         {'action': 'store_true', 'default':False,
          'metavar': "<yN>",
          'help':'include cubicweb user entities in schema',
          }),
        ('exclude-type',
         {'type':'string', 'default':'',
          'short': "x", 'metavar': "<types>",
          'help':'coma separated list of entity types to remove from view',
          }),
        ('include-type',
         {'type':'string', 'default':'',
          'short': "i", 'metavar': "<types>",
          'help':'coma separated list of entity types to include in view',
          }),
792
793
794
795
796
        ('show-etype',
         {'type':'string', 'default':'',
          'metavar': '<etype>',
          'help':'show graph of this etype and its neighbours'
          }),
797
        ]
798
799

    def run(self, args):
800
801
        from subprocess import Popen
        from tempfile import NamedTemporaryFile
802
        from logilab.common.textutils import splitstrip
803
804
        from logilab.common.graph import GraphGenerator, DotBackend
        from yams import schema2dot as s2d, BASE_TYPES
805
806
        from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,
                                     WORKFLOW_TYPES, INTERNAL_TYPES)
807
        cubes = splitstrip(args[0])
808
        dev_conf = DevConfiguration(*cubes)
809
810
811
812
813
        schema = dev_conf.load_schema()
        out, viewer = self['output-file'], self['viewer']
        if out is None:
            tmp_file = NamedTemporaryFile(suffix=".svg")
            out = tmp_file.name
814
815
816
817
818
819
820
821
822
        skiptypes = BASE_TYPES | SCHEMA_TYPES
        if not self['show-meta']:
            skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES
        if not self['show-workflow']:
            skiptypes |= WORKFLOW_TYPES
        if not self['show-cw-user']:
            skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))
        skiptypes |= set(self['exclude-type'].split(','))
        skiptypes -= set(self['include-type'].split(','))
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838

        if not self['show-etype']:
            s2d.schema2dot(schema, out, skiptypes=skiptypes)
        else:
            etype = self['show-etype']
            visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)
            propshdlr = s2d.SchemaDotPropsHandler(visitor)
            backend = DotBackend('schema', 'BT',
                                 ratio='compress',size=None,
                                 renderer='dot',
                                 additionnal_param={'overlap' : 'false',
                                                    'splines' : 'true',
                                                    'sep' : '0.2'})
            generator = s2d.GraphGenerator(backend)
            generator.generate(visitor, propshdlr, out)

839
840
841
842
        if viewer:
            p = Popen((viewer, out))
            p.wait()

843

844
for cmdcls in (UpdateCubicWebCatalogCommand,
845
               UpdateCubeCatalogCommand,
846
847
848
849
850
851
               #LiveServerCommand,
               NewCubeCommand,
               ExamineLogCommand,
               GenerateSchema,
               ):
    CWCTL.register(cmdcls)