devctl.py 38.2 KB
Newer Older
1
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2
# contact https://www.logilab.fr/ -- mailto:contact@logilab.fr
3
4
5
6
7
8
9
10
#
# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
12
13
14
15
16
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
17
# with CubicWeb.  If not, see <https://www.gnu.org/licenses/>.
18
19
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
Adrien Di Mascio's avatar
Adrien Di Mascio committed
20
"""
21
22
23
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
24
25
26

import shutil
import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
27
import sys
28
from datetime import datetime, date
29
from os import getcwd, mkdir, chdir, path as osp
30
import pkg_resources
Adrien Di Mascio's avatar
Adrien Di Mascio committed
31

32
from pytz import UTC
33

Adrien Di Mascio's avatar
Adrien Di Mascio committed
34
from logilab.common import STD_BLACKLIST
35
from logilab.common.modutils import clean_sys_modules
36
37
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find
38

Adrien Di Mascio's avatar
Adrien Di Mascio committed
39
from cubicweb.__pkginfo__ import version as cubicwebversion
40
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
41
from cubicweb.i18n import extract_from_tal, execute2
42
from cubicweb.cwctl import CWCTL
43
from cubicweb.cwconfig import CubicWebNoAppConfiguration
44
from cubicweb.toolsutils import SKEL_EXCLUDE, Command, copy_skeleton, underline_title
Adrien Di Mascio's avatar
Adrien Di Mascio committed
45
46
47
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration

48

49
50
51
__docformat__ = "restructuredtext en"


52
STD_BLACKLIST = set(STD_BLACKLIST)
53
54
55
STD_BLACKLIST.add(".tox")
STD_BLACKLIST.add("test")
STD_BLACKLIST.add("node_modules")
56
57


58
59
60
61
class DevConfiguration(ServerConfiguration, WebConfiguration):
    """dummy config to get full library schema and appobjects for
    a cube or for cubicweb (without a home)
    """
62

Adrien Di Mascio's avatar
Adrien Di Mascio committed
63
    creating = True
64
    cleanup_unused_appobjects = False
65

66
67
68
69
70
71
72
    cubicweb_appobject_path = (
        ServerConfiguration.cubicweb_appobject_path
        | WebConfiguration.cubicweb_appobject_path
    )
    cube_appobject_path = (
        ServerConfiguration.cube_appobject_path | WebConfiguration.cube_appobject_path
    )
Sylvain Thenault's avatar
Sylvain Thenault committed
73

74
    def __init__(self, *cubes):
75
76
77
        super(DevConfiguration, self).__init__(cubes and cubes[0] or None)
        if cubes:
            self._cubes = self.reorder_cubes(
78
79
                self.expand_cubes(cubes, with_recommends=True)
            )
80
            self.load_site_cubicweb()
81
82
        else:
            self._cubes = ()
83

Adrien Di Mascio's avatar
Adrien Di Mascio committed
84
85
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
86
        return None
87
88
89
90

    def available_languages(self):
        return self.cw_languages()

Sylvain Thenault's avatar
Sylvain Thenault committed
91
92
    def main_config_file(self):
        return None
93

94
    def init_log(self):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
95
        pass
96

97
    def load_configuration(self, **kw):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
98
        pass
99

Sylvain Thenault's avatar
Sylvain Thenault committed
100
101
    def default_log_file(self):
        return None
102

103
104
    def default_stats_file(self):
        return None
Sylvain Thenault's avatar
Sylvain Thenault committed
105
106


Adrien Di Mascio's avatar
Adrien Di Mascio committed
107
108
109
110
111
112
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
113
    from cubicweb.cwvreg import CWRegistryStore
114

Adrien Di Mascio's avatar
Adrien Di Mascio committed
115
    if cubedir:
116
        cube = osp.split(cubedir)[-1]
117
118
        if cube.startswith("cubicweb_"):
            cube = cube[len("cubicweb_") :]
119
120
121
122
        config = DevConfiguration(cube)
        depcubes = list(config._cubes)
        depcubes.remove(cube)
        libconfig = DevConfiguration(*depcubes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
123
    else:
124
125
        config = DevConfiguration()
        cube = libconfig = None
126
    clean_sys_modules(config.appobjects_modnames())
127
    schema = config.load_schema(remove_unused_rtypes=False)
128
    vreg = CWRegistryStore(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
129
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
130
131
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
132
    _generate_schema_pot(w, vreg, schema, libconfig=libconfig)
133
134


135
def _generate_schema_pot(w, vreg, schema, libconfig=None):
136
    from cubicweb.i18n import add_msg
137
    from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
138
139
140
141
142
143
144
145

    w(
        "# schema pot file, generated on %s\n"
        % datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
    )
    w("# \n")
    w("# singular and plural forms for each entity type\n")
    w("\n")
146
    vregdone = set()
147
148
    afss = vreg["uicfg"]["autoform_section"]
    aiams = vreg["uicfg"]["actionbox_appearsin_addmenu"]
149
    if libconfig is not None:
150
151
        # processing a cube, libconfig being a config with all its dependencies
        # (cubicweb incl.)
152
        from cubicweb.cwvreg import CWRegistryStore
153

154
        libschema = libconfig.load_schema(remove_unused_rtypes=False)
155
        clean_sys_modules(libconfig.appobjects_modnames())
156
        libvreg = CWRegistryStore(libconfig)
157
158
159
        libvreg.set_schema(libschema)  # trigger objects registration
        libafss = libvreg["uicfg"]["autoform_section"]
        libaiams = libvreg["uicfg"]["actionbox_appearsin_addmenu"]
160
161
        # prefill vregdone set
        list(_iter_vreg_objids(libvreg, vregdone))
162
163

        def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):
164
165
166
167
168
            return any(
                predicate(rtag.etype_get(eschema, rschema, role, tschema))
                for rtag in rtags
            )

Adrien Di Mascio's avatar
Adrien Di Mascio committed
169
    else:
170
        # processing cubicweb itself
171
        libschema = {}
172
173
        for cstrtype in CONSTRAINTS:
            add_msg(w, cstrtype)
174
175
        libafss = libaiams = None
        is_in_lib = lambda *args: False
Adrien Di Mascio's avatar
Adrien Di Mascio committed
176
    done = set()
177
178
179
    for eschema in sorted(schema.entities()):
        if eschema.type in libschema:
            done.add(eschema.description)
180
    for eschema in sorted(schema.entities()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
181
        etype = eschema.type
182
183
        if etype not in libschema:
            add_msg(w, etype)
184
            add_msg(w, "%s_plural" % etype)
185
            if not eschema.final:
186
187
188
                add_msg(w, "This %s:" % etype)
                add_msg(w, "New %s" % etype)
                add_msg(w, "add a %s" % etype)  # AddNewAction
189
190
191
192
                if libconfig is not None:  # processing a cube
                    # As of 3.20.3 we no longer use it, but keeping this string
                    # allows developers to run i18ncube with new cubicweb and still
                    # have the right translations at runtime for older versions
193
                    add_msg(w, "This %s" % etype)
194
195
196
            if eschema.description and not eschema.description in done:
                done.add(eschema.description)
                add_msg(w, eschema.description)
197
        if eschema.final:
198
199
            continue
        for rschema, targetschemas, role in eschema.relation_definitions(True):
200
201
            if rschema.final:
                continue
202
            for tschema in targetschemas:
203

204
205
                for afs in afss:
                    fsections = afs.etype_get(eschema, rschema, role, tschema)
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
                    if "main_inlined" in fsections and not is_in_lib(
                        libafss,
                        eschema,
                        rschema,
                        role,
                        tschema,
                        lambda x: "main_inlined" in x,
                    ):
                        add_msg(
                            w,
                            "add a %s" % tschema,
                            "inlined:%s.%s.%s" % (etype, rschema, role),
                        )
                        add_msg(
                            w, str(tschema), "inlined:%s.%s.%s" % (etype, rschema, role)
                        )
222
223
224
                        break

                for aiam in aiams:
225
226
227
228
229
230
231
232
233
234
                    if aiam.etype_get(
                        eschema, rschema, role, tschema
                    ) and not is_in_lib(libaiams, eschema, rschema, role, tschema):
                        if role == "subject":
                            label = "add %s %s %s %s" % (
                                eschema,
                                rschema,
                                tschema,
                                role,
                            )
235
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (
236
237
238
239
240
                                tschema,
                                eschema,
                                rschema,
                                tschema,
                            )
241
                        else:
242
243
244
245
246
247
                            label = "add %s %s %s %s" % (
                                tschema,
                                rschema,
                                eschema,
                                role,
                            )
248
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (
249
250
251
252
253
                                tschema,
                                tschema,
                                rschema,
                                eschema,
                            )
254
255
256
                        add_msg(w, label)
                        add_msg(w, label2)
                        break
Sylvain Thénault's avatar
Sylvain Thénault committed
257
258
            # XXX also generate "creating ...' messages for actions in the
            # addrelated submenu
259
260
261
    w("# subject and object forms for each relation type\n")
    w("# (no object form for final or symmetric relation types)\n")
    w("\n")
262
263
264
265
    for rschema in sorted(schema.relations()):
        if rschema.type in libschema:
            done.add(rschema.type)
            done.add(rschema.description)
266
    for rschema in sorted(schema.relations()):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
267
        rtype = rschema.type
268
        if rtype not in libschema:
269
270
            # bw compat, necessary until all translation of relation are done
            # properly...
271
            add_msg(w, rtype)
272
            done.add(rtype)
273
274
            if rschema.description and rschema.description not in done:
                add_msg(w, rschema.description)
275
            done.add(rschema.description)
276
277
278
            librschema = None
        else:
            librschema = libschema.rschema(rtype)
279
        # add context information only for non-metadata rtypes
280
        if rschema not in NO_I18NCONTEXT:
281
            libsubjects = librschema and librschema.subjects() or ()
282
            for subjschema in rschema.subjects():
283
284
                if not subjschema in libsubjects:
                    add_msg(w, rtype, subjschema.type)
285
        if not (rschema.final or rschema.symmetric):
286
            if rschema not in NO_I18NCONTEXT:
287
288
289
                libobjects = librschema and librschema.objects() or ()
                for objschema in rschema.objects():
                    if not objschema in libobjects:
290
                        add_msg(w, "%s_object" % rtype, objschema.type)
291
            if rtype not in libschema:
292
293
                # bw compat, necessary until all translation of relation are
                # done properly...
294
                add_msg(w, "%s_object" % rtype)
295
        for rdef in rschema.rdefs.values():
296
297
            if not rdef.description or rdef.description in done:
                continue
298
299
300
301
302
303
            if (
                librschema is None
                or (rdef.subject, rdef.object) not in librschema.rdefs
                or librschema.rdefs[(rdef.subject, rdef.object)].description
                != rdef.description
            ):
304
305
                add_msg(w, rdef.description)
            done.add(rdef.description)
306
    for objid in _iter_vreg_objids(vreg, vregdone):
307
        add_msg(w, "%s_description" % objid)
308
309
        add_msg(w, objid)

Sylvain Thénault's avatar
Sylvain Thénault committed
310

311
def _iter_vreg_objids(vreg, done):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
312
    for reg, objdict in vreg.items():
313
        if reg in ("boxes", "contentnavigation"):
314
            continue
315
        for objects in objdict.values():
Adrien Di Mascio's avatar
Adrien Di Mascio committed
316
            for obj in objects:
317
                objid = "%s_%s" % (reg, obj.__regid__)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
318
                if objid in done:
319
                    break
320
                pdefs = getattr(obj, "cw_property_defs", {})
Sylvain Thénault's avatar
Sylvain Thénault committed
321
                if pdefs:
322
                    yield objid
Adrien Di Mascio's avatar
Adrien Di Mascio committed
323
                    done.add(objid)
324
                    break
325

326

327
328
DEFAULT_POT_HEAD = (
    r"""msgid ""
Adrien Di Mascio's avatar
Adrien Di Mascio committed
329
330
331
332
333
334
335
336
337
338
339
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

340
341
342
"""
    % cubicwebversion
)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
343
344
345
346


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
347

Adrien Di Mascio's avatar
Adrien Di Mascio committed
348
349
350
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
351
352

    name = "i18ncubicweb"
353
    min_args = max_args = 0
Adrien Di Mascio's avatar
Adrien Di Mascio committed
354
355
356
357

    def run(self, args):
        """run the command with its specific arguments"""
        import shutil
358
        import tempfile
Adrien Di Mascio's avatar
Adrien Di Mascio committed
359
360
        import yams
        from logilab.common.fileutils import ensure_fs_mode
361
        from logilab.common.shellutils import globfind, find, rm
362
        from logilab.common.modutils import get_module_files
363
        from cubicweb.i18n import extract_from_tal, execute2
364
365

        tempdir = tempfile.mkdtemp(prefix="cw-")
366
        cwi18ndir = WebConfiguration.i18n_lib_dir()
367
368
369
        print("-> extract messages:", end=" ")
        print("schema", end=" ")
        schemapot = osp.join(tempdir, "schema.pot")
370
        potfiles = [schemapot]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
371
372
373
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
374
        schemapotstream = open(schemapot, "w")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
375
376
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
377
378
379
380
        print("TAL", end=" ")
        tali18nfile = osp.join(tempdir, "tali18n.py")
        extract_from_tal(find(osp.join(BASEDIR, "web"), (".py", ".pt")), tali18nfile)
        print("-> generate .pot files.")
381
        pyfiles = get_module_files(BASEDIR)
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
        pyfiles += globfind(osp.join(BASEDIR, "misc", "migration"), "*.py")
        schemafiles = globfind(osp.join(BASEDIR, "schemas"), "*.py")
        jsfiles = globfind(osp.join(BASEDIR, "web"), "cub*.js")
        for id, files, lang in [
            ("pycubicweb", pyfiles, None),
            ("schemadescr", schemafiles, None),
            ("yams", get_module_files(yams.__path__[0]), None),
            ("tal", [tali18nfile], None),
            ("js", jsfiles, "java"),
        ]:
            potfile = osp.join(tempdir, "%s.pot" % id)
            cmd = [
                "xgettext",
                "--from-code=UTF-8",
                "--no-location",
                "--omit-header",
                "-k_",
            ]
400
            if lang is not None:
401
402
                cmd.extend(["-L", lang])
            cmd.extend(["-o", potfile])
403
404
            cmd.extend(files)
            execute2(cmd)
405
            if osp.exists(potfile):
406
407
                potfiles.append(potfile)
            else:
408
409
410
411
                print("-> WARNING: %s file was not generated" % potfile)
        print("-> merging %i .pot files" % len(potfiles))
        cubicwebpot = osp.join(tempdir, "cubicweb.pot")
        cmd = ["msgcat", "-o", cubicwebpot] + potfiles
412
        execute2(cmd)
413
        print("-> merging main pot file with existing translations.")
414
        chdir(cwi18ndir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
415
        toedit = []
416
        for lang in CubicWebNoAppConfiguration.cw_languages():
417
418
419
420
421
422
423
424
425
426
            target = "%s.po" % lang
            cmd = [
                "msgmerge",
                "-N",
                "--sort-output",
                "-o",
                target + "new",
                target,
                cubicwebpot,
            ]
427
            execute2(cmd)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
428
            ensure_fs_mode(target)
429
            shutil.move("%snew" % target, target)
430
            toedit.append(osp.abspath(target))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
431
        # cleanup
432
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
433
        # instructions pour la suite
434
435
436
        print("-> regenerated CubicWeb's .po catalogs.")
        print("\nYou can now edit the following files:")
        print("* " + "\n* ".join(toedit))
Samuel Trégouët's avatar
Samuel Trégouët committed
437
        print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
438
439


440
class UpdateCubeCatalogCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
441
442
443
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
444
445
446

    name = "i18ncube"
    arguments = "[<cube>...]"
447

Adrien Di Mascio's avatar
Adrien Di Mascio committed
448
449
450
    def run(self, args):
        """run the command with its specific arguments"""
        if args:
451
            cubes = [DevConfiguration.cube_dir(cube) for cube in args]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
452
        else:
453
454
455
456
457
458
459
            cubes = [
                DevConfiguration.cube_dir(cube)
                for cube in DevConfiguration.available_cubes()
            ]
            cubes = [
                cubepath for cubepath in cubes if osp.exists(osp.join(cubepath, "i18n"))
            ]
460
461
        if not update_cubes_catalogs(cubes):
            raise ExecutionError("update cubes i18n catalog failed")
Adrien Di Mascio's avatar
Adrien Di Mascio committed
462

463

Adrien Di Mascio's avatar
Adrien Di Mascio committed
464
def update_cubes_catalogs(cubes):
465
    from subprocess import CalledProcessError
466

Adrien Di Mascio's avatar
Adrien Di Mascio committed
467
    for cubedir in cubes:
468
        if not osp.isdir(cubedir):
469
            print("-> ignoring %s that is not a directory." % cubedir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
470
            continue
471
        try:
472
            toedit = update_cube_catalogs(cubedir)
473
        except CalledProcessError as exc:
474
475
476
            print("\n*** error while updating catalogs for cube", cubedir)
            print("cmd:\n%s" % exc.cmd)
            print("stdout:\n%s\nstderr:\n%s" % exc.data)
477
478
        except Exception:
            import traceback
479

480
            traceback.print_exc()
481
            print("*** error while updating catalogs for cube", cubedir)
482
            return False
Sylvain Thénault's avatar
Sylvain Thénault committed
483
484
        else:
            # instructions pour la suite
485
            if toedit:
486
487
488
489
490
491
492
                print("-> regenerated .po catalogs for cube %s." % cubedir)
                print("\nYou can now edit the following files:")
                print("* " + "\n* ".join(toedit))
                print(
                    'When you are done, run "cubicweb-ctl i18ninstance '
                    '<yourinstance>" to see changes in your instances.'
                )
493
            return True
Adrien Di Mascio's avatar
Adrien Di Mascio committed
494

495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510

class I18nCubeMessageExtractor(object):
    """This class encapsulates all the xgettext extraction logic

    ``generate_pot_file`` is the main entry point called by the ``i18ncube``
    command. A cube might decide to customize extractors to ignore a given
    directory or to extract messages from a new file type (e.g. .jinja2 files)

    For each file type, the class must define two methods:

    - ``collect_{filetype}()`` that must return the list of files
      xgettext should inspect,

    - ``extract_{filetype}(files)`` that calls xgettext and returns the
      path to the generated ``pot`` file
    """
511

512
    blacklist = STD_BLACKLIST
513
    formats = ["tal", "js", "py"]
514
515
516
517
518
519
520
521
522
523
524
525
526

    def __init__(self, workdir, cubedir):
        self.workdir = workdir
        self.cubedir = cubedir

    def generate_pot_file(self):
        """main entry point: return the generated ``cube.pot`` file

        This function first generates all the pot files (schema, tal,
        py, js) and then merges them in a single ``cube.pot`` that will
        be used to eventually update the ``i18n/*.po`` files.
        """
        potfiles = self.generate_pot_files()
527
528
529
        potfile = osp.join(self.workdir, "cube.pot")
        print("-> merging %i .pot files" % len(potfiles))
        cmd = ["msgcat", "-o", potfile]
530
531
532
533
534
        cmd.extend(potfiles)
        execute2(cmd)
        return potfile if osp.exists(potfile) else None

    def find(self, exts, blacklist=None):
535
        """collect files with extensions ``exts`` in the cube directory"""
536
537
538
539
540
541
542
543
544
545
546
        if blacklist is None:
            blacklist = self.blacklist
        return find(self.cubedir, exts, blacklist=blacklist)

    def generate_pot_files(self):
        """generate and return the list of all ``pot`` files for the cube

        - static-messages.pot,
        - schema.pot,
        - one ``pot`` file for each inspected format (.py, .js, etc.)
        """
547
        print("-> extracting messages:", end=" ")
548
        potfiles = []
549
        # static messages
550
551
        if osp.exists(osp.join("i18n", "static-messages.pot")):
            potfiles.append(osp.join("i18n", "static-messages.pot"))
552
553
554
555
        # messages from schema
        potfiles.append(self.schemapot())
        # messages from sourcecode
        for fmt in self.formats:
556
557
            collector = getattr(self, "collect_{}".format(fmt))
            extractor = getattr(self, "extract_{}".format(fmt))
558
559
560
561
562
563
564
565
566
            files = collector()
            if files:
                potfile = extractor(files)
                if potfile:
                    potfiles.append(potfile)
        return potfiles

    def schemapot(self):
        """generate the ``schema.pot`` file"""
567
568
        schemapot = osp.join(self.workdir, "schema.pot")
        print("schema", end=" ")
569
570
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
571
        schemapotstream = open(schemapot, "w")
572
573
574
575
        generate_schema_pot(schemapotstream.write, self.cubedir)
        schemapotstream.close()
        return schemapot

576
577
    def _xgettext(self, files, output, k="_", extraopts=""):
        """shortcut to execute the xgettext command and return output file"""
578
        tmppotfile = osp.join(self.workdir, output)
579
580
581
582
583
584
585
586
587
588
589
590
591
        cmd = (
            [
                "xgettext",
                "--from-code=UTF-8",
                "--no-location",
                "--omit-header",
                "-k" + k,
                "-o",
                tmppotfile,
            ]
            + extraopts.split()
            + files
        )
592
        execute2(cmd)
593
        if osp.exists(tmppotfile):
594
595
596
            return tmppotfile

    def collect_tal(self):
597
598
        print("TAL", end=" ")
        return self.find((".py", ".pt"))
599
600

    def extract_tal(self, files):
601
        tali18nfile = osp.join(self.workdir, "tali18n.py")
602
        extract_from_tal(files, tali18nfile)
603
        return self._xgettext(files, output="tal.pot")
604
605

    def collect_js(self):
606
607
608
609
610
611
        print("Javascript")
        return [
            jsfile
            for jsfile in self.find(".js")
            if osp.basename(jsfile).startswith("cub")
        ]
612
613

    def extract_js(self, files):
614
615
616
        return self._xgettext(
            files, output="js.pot", extraopts="-L java --from-code=utf-8"
        )
617
618

    def collect_py(self):
619
620
        print("-> creating cube-specific catalog")
        return self.find(".py")
621
622

    def extract_py(self, files):
623
        return self._xgettext(files, output="py.pot")
624
625
626
627
628


def update_cube_catalogs(cubedir):
    cubedir = osp.abspath(osp.normpath(cubedir))
    workdir = tempfile.mkdtemp()
629
    try:
630
        cubename = osp.basename(cubedir)
631
        if cubename.startswith("cubicweb_"):  # new layout
632
            distname = cubename
633
            cubename = cubename[len("cubicweb_") :]
634
        else:
635
636
            distname = "cubicweb_" + cubename
        print("cubedir", cubedir)
637
638
639
        extract_cls = I18nCubeMessageExtractor
        try:
            extract_cls = pkg_resources.load_entry_point(
640
641
                distname, "cubicweb.i18ncube", cubename
            )
642
643
        except (pkg_resources.DistributionNotFound, ImportError):
            pass  # no customization found
644
        print(underline_title("Updating i18n catalogs for cube %s" % cubename))
645
        chdir(cubedir)
646
        extractor = extract_cls(workdir, cubedir)
647
648
        potfile = extractor.generate_pot_file()
        if potfile is None:
649
            print("no message catalog for cube", cubename, "nothing to translate")
650
            return ()
651
652
        print("-> merging main pot file with existing translations:", end=" ")
        chdir("i18n")
653
654
        toedit = []
        for lang in CubicWebNoAppConfiguration.cw_languages():
655
656
            print(lang, end=" ")
            cubepo = "%s.po" % lang
657
658
659
            if not osp.exists(cubepo):
                shutil.copy(potfile, cubepo)
            else:
660
                cmd = ["msgmerge", "-N", "-s", "-o", cubepo + "new", cubepo, potfile]
661
662
                execute2(cmd)
                ensure_fs_mode(cubepo)
663
                shutil.move("%snew" % cubepo, cubepo)
664
665
666
667
668
            toedit.append(osp.abspath(cubepo))
        print()
        return toedit
    finally:
        # cleanup
669
        shutil.rmtree(workdir)
670
671


672
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
673
674
675
    """Create a new cube.

    <cubename>
676
      the name of the new cube. It should be a valid python module name.
Adrien Di Mascio's avatar
Adrien Di Mascio committed
677
    """
678
679
680

    name = "newcube"
    arguments = "<cubename>"
681
    min_args = max_args = 1
682
    options = (
683
684
685
686
687
688
689
690
691
        (
            "layout",
            {
                "short": "L",
                "type": "choice",
                "metavar": "<cube layout>",
                "default": "simple",
                "choices": ("simple", "full"),
                "help": 'cube layout. You\'ll get a minimal cube with the "simple" \
692
layout, and a full featured cube with "full" layout.',
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
            },
        ),
        (
            "directory",
            {
                "short": "d",
                "type": "string",
                "metavar": "<cubes directory>",
                "help": "directory where the new cube should be created",
            },
        ),
        (
            "verbose",
            {
                "short": "v",
                "type": "yn",
                "metavar": "<verbose>",
                "default": "n",
                "help": "verbose mode: will ask all possible configuration questions",
            },
        ),
        (
            "short-description",
            {
                "short": "s",
                "type": "string",
                "metavar": "<short description>",
                "help": "short description for the cube",
            },
        ),
        (
            "author",
            {
                "short": "a",
                "type": "string",
                "metavar": "<author>",
                "default": "LOGILAB S.A. (Paris, FRANCE)",
                "help": "cube author",
            },
        ),
        (
            "author-email",
            {
                "short": "e",
                "type": "string",
                "metavar": "<email>",
                "default": "contact@logilab.fr",
                "help": "cube author's email",
            },
        ),
        (
            "author-web-site",
            {
                "short": "w",
                "type": "string",
                "metavar": "<web site>",
749
                "default": "https://www.logilab.fr",
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
                "help": "cube author's web site",
            },
        ),
        (
            "license",
            {
                "short": "l",
                "type": "choice",
                "metavar": "<license>",
                "default": "LGPL",
                "choices": ("GPL", "LGPL", ""),
                "help": "cube license",
            },
        ),
    )
765

766
    LICENSES = {
767
        "LGPL": """\
768
769
770
771
772
773
774
775
776
777
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
778
# You should have received a copy of the GNU Lesser General Public License
779
# along with this program. If not, see <https://www.gnu.org/licenses/>.
780
781
""",
        "GPL": """\
782
783
784
785
786
787
788
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 2.1 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
789
790
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
791
792
#
# You should have received a copy of the GNU General Public License along with
793
# this program. If not, see <https://www.gnu.org/licenses/>.
794
795
796
""",
        "": "# INSERT LICENSE HERE",
    }
797

Adrien Di Mascio's avatar
Adrien Di Mascio committed
798
    def run(self, args):
799
        import re
800
        from logilab.common.shellutils import ASK
801

802
        cubename = args[0]
803
804
805
806
        if not re.match("[_A-Za-z][_A-Za-z0-9]*$", cubename):
            raise BadCommandUsage("cube name must be a valid python module name")
        verbose = self.get("verbose")
        destdir = self.get("directory")
807
        if not destdir:
808
809
            destdir = getcwd()
        elif not osp.isdir(destdir):
810
            print("-> creating cubes directory", destdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
811
            try:
812
                mkdir(destdir)
813
            except OSError as err:
814
815
                self.fail("failed to create directory %r\n(%s)" % (destdir, err))
        default_name = "cubicweb-%s" % cubename.lower().replace("_", "-")
816
        if verbose:
817
818
819
            distname = input(
                "Debian name for your cube ? [%s]): " % default_name
            ).strip()
820
            if not distname:
821
                distname = default_name
822
823
824
            elif not distname.startswith("cubicweb-"):
                if ASK.confirm("Do you mean cubicweb-%s ?" % distname):
                    distname = "cubicweb-" + distname
825
        else:
826
            distname = default_name
827
828
        if not re.match("[a-z][-a-z0-9]*$", distname):
            raise BadCommandUsage("cube distname should be a valid debian package name")
829
830
831
        cubedir = osp.join(destdir, distname)
        if osp.exists(cubedir):
            self.fail("%s already exists!" % cubedir)
832
833
834
835
        skeldir = osp.join(BASEDIR, "skeleton")
        longdesc = shortdesc = self["short-description"] or input(
            "Enter a short description for your cube: "
        )
836
        if verbose:
837
            longdesc = input(
838
839
                "Enter a long description (leave empty to reuse the short one): "
            )
840
        dependencies = {
841
            "cubicweb": ">= %s" % cubicwebversion,
842
        }
843
        if verbose:
844
            dependencies.update(self._ask_for_dependencies())
845
        context = {
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
            "cubename": cubename,
            "distname": distname,
            "shortdesc": shortdesc,
            "longdesc": longdesc or shortdesc,
            "dependencies": str(dependencies).replace(
                "'", '"'
            ),  # to respect black formatting
            "version": cubicwebversion,
            "year": str(date.today().year),
            "author": self["author"],
            "author-email": self["author-email"],
            "rfc2822-date": datetime.now(tz=UTC).strftime("%a, %d %b %Y %T %z"),
            "author-web-site": self["author-web-site"],
            "license": self["license"],
            "long-license": self.LICENSES[self["license"]],
861
        }
862
        exclude = SKEL_EXCLUDE
863
864
865
866
867
868
869
870
        if self["layout"] == "simple":
            exclude += (
                "sobjects.py*",
                "precreate.py*",
                "realdb_test*",
                "cubes.*",
                "uiprops.py*",
            )
871
        copy_skeleton(skeldir, cubedir, context, exclude=exclude)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
872

873
    def _ask_for_dependencies(self):
874
875
        from logilab.common.shellutils import ASK
        from logilab.common.textutils import splitstrip
876

877
878
        depcubes = []
        for cube in ServerConfiguration.available_cubes():
879
880
881
882
            answer = ASK.ask(
                "Depends on cube %s? " % cube, ("N", "y", "skip", "type"), "N"
            )
            if answer == "y":
883
                depcubes.append(cube)
884
885
            if answer == "type":
                depcubes = splitstrip(input("type dependencies: "))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
886
                break
887
            elif answer == "skip":
Adrien Di Mascio's avatar
Adrien Di Mascio committed
888
                break
889
890
891
892
        return dict(
            ("cubicweb-" + cube, ServerConfiguration.cube_version(cube))
            for cube in depcubes
        )
893

894
895
896
897

class ExamineLogCommand(Command):
    """Examine a rql log file.

898
    Will print out the following table
899

900
      Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query
901

902
    sorted by descending cumulative time (clock). Time are expressed in seconds.
903

904
    Chances are the lines at the top are the ones that will bring the higher
905
    benefit after optimisation. Start there.
906
    """
907
908
909

    arguments = "rql.log"
    name = "exlog"
910
    options = ()
911

912
913
    def run(self, args):
        import re
914

915
        requests = {}
916
        for filepath in args:
917
            try:
Rémi Cardona's avatar
Rémi Cardona committed
918
                stream = open(filepath)
919
            except OSError as ex:
920
                raise BadCommandUsage("can't open rql log file %s: %s" % (filepath, ex))
921
            for lineno, line in enumerate(stream):
922
                if not " WHERE " in line:
923
924
                    continue
                try:
925
926
927
928
                    rql, time = line.split("--")
                    rql = re.sub(r"(\'\w+': \d*)", "", rql)
                    if "{" in rql:
                        rql = rql[: rql.index("{")]
929
930
931
932
933
                    req = requests.setdefault(rql, [])
                    time.strip()
                    chunks = time.split()
                    clocktime = float(chunks[0][1:])
                    cputime = float(chunks[-3])
934
                    req.append((clocktime, cputime))
935
                except Exception as exc:
936
                    sys.stderr.write("Line %s: %s (%s)\n" % (lineno, exc, line))
937
        stat = []
938
        for rql, times in requests.items():
939
940
941
942
943
944
945
946
            stat.append(
                (
                    sum(time[0] for time in times),
                    sum(time[1] for time in times),
                    len(times),
                    rql,
                )
            )
947
948
        stat.sort()
        stat.reverse()
949
        total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
950
951
952
        print(
            "Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query"
        )
953
        for clocktime, cputime, occ, rql in stat:
954
955
956
957
            print(
                "%.2f;%.2f;%.2f;%s;%s"
                % (clocktime / total_time, clocktime, cputime, occ, rql)
            )
958

959

960
961
class GenerateSchema(Command):
    """Generate schema image for the given cube"""
962

963
    name = "schema"
964
    arguments = "<cube>"
965
    min_args = max_args = 1
966
    options = [
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985