devctl.py 18.7 KB
Newer Older
Adrien Di Mascio's avatar
Adrien Di Mascio committed
1
2
3
4
5
6
7
8
9
10
"""additional cubicweb-ctl commands and command handlers for cubicweb and cubicweb's
cubes development

:organization: Logilab
:copyright: 2001-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
"""
__docformat__ = "restructuredtext en"

import sys
Sylvain Thenault's avatar
Sylvain Thenault committed
11
from os import walk, mkdir, chdir, listdir, getcwd
Adrien Di Mascio's avatar
Adrien Di Mascio committed
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
from os.path import join, exists, abspath, basename, normpath, split, isdir


from logilab.common import STD_BLACKLIST
from logilab.common.modutils import get_module_files
from logilab.common.textutils import get_csv

from cubicweb import CW_SOFTWARE_ROOT as BASEDIR
from cubicweb.__pkginfo__ import version as cubicwebversion
from cubicweb import BadCommandUsage
from cubicweb.toolsutils import Command, register_commands, confirm, copy_skeleton
from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration


Sylvain Thenault's avatar
Sylvain Thenault committed
27
class DevCubeConfiguration(ServerConfiguration, WebConfiguration):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
28
29
    """dummy config to get full library schema and entities"""
    creating = True
Sylvain Thenault's avatar
Sylvain Thenault committed
30
31
32
33
34
35
    cubicweb_vobject_path = ServerConfiguration.cubicweb_vobject_path | WebConfiguration.cubicweb_vobject_path
    cube_vobject_path = ServerConfiguration.cube_vobject_path | WebConfiguration.cube_vobject_path

    def __init__(self, cube):
        super(DevCubeConfiguration, self).__init__(cube)
        if cube is None:
Adrien Di Mascio's avatar
Adrien Di Mascio committed
36
37
            self._cubes = ()
        else:
Sylvain Thenault's avatar
Sylvain Thenault committed
38
            self._cubes = self.expand_cubes((cube,))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
39
40
41
        
    @property
    def apphome(self):
Sylvain Thenault's avatar
Sylvain Thenault committed
42
43
44
        return None
    def main_config_file(self):
        return None
Adrien Di Mascio's avatar
Adrien Di Mascio committed
45
46
47
48
49
50
    def init_log(self, debug=None):
        pass
    def load_configuration(self):
        pass


Sylvain Thenault's avatar
Sylvain Thenault committed
51
52
53
54
55
56
57
58
59
60
class DevDepConfiguration(DevCubeConfiguration):
    """configuration to use to generate cubicweb po files or to use as "library" configuration
    to filter out message ids from cubicweb and dependencies of a cube
    """
    def __init__(self, cube=None):
        super(DevDepConfiguration, self).__init__(cube)
        if cube is None:
            self._cubes = ()
        else:
            self._cubes = self.expand_cubes(self.cube_dependencies(cube))
Adrien Di Mascio's avatar
Adrien Di Mascio committed
61

Sylvain Thenault's avatar
Sylvain Thenault committed
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    def default_log_file(self):
        return None


def cleanup_sys_modules(config):
    # cleanup sys.modules, required when we're updating multiple cubes
    for name, mod in sys.modules.items():
        if mod is None:
            # duh ? logilab.common.os for instance
            del sys.modules[name]
            continue
        if not hasattr(mod, '__file__'):
            continue
        for path in config.vregistry_path():
            if mod.__file__.startswith(path):
                del sys.modules[name]
                break
    
Adrien Di Mascio's avatar
Adrien Di Mascio committed
80
81
82
83
84
85
86
87
def generate_schema_pot(w, cubedir=None):
    """generate a pot file with schema specific i18n messages

    notice that relation definitions description and static vocabulary
    should be marked using '_' and extracted using xgettext
    """
    from cubicweb.cwvreg import CubicWebRegistry
    cube = cubedir and split(cubedir)[-1]
Sylvain Thenault's avatar
Sylvain Thenault committed
88
89
    config = DevDepConfiguration(cube)
    cleanup_sys_modules(config)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
90
91
    if cubedir:
        libschema = config.load_schema()
Sylvain Thenault's avatar
Sylvain Thenault committed
92
        config = DevCubeConfiguration(cube)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
93
94
95
96
97
98
        schema = config.load_schema()
    else:
        schema = config.load_schema()
        libschema = None
        config.cleanup_interface_sobjects = False
    vreg = CubicWebRegistry(config)
Sylvain Thenault's avatar
Sylvain Thenault committed
99
    # set_schema triggers objects registrations
Adrien Di Mascio's avatar
Adrien Di Mascio committed
100
101
    vreg.set_schema(schema)
    w(DEFAULT_POT_HEAD)
Sylvain Thenault's avatar
Sylvain Thenault committed
102
    _generate_schema_pot(w, vreg, schema, libschema=libschema, cube=cube)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
                
def _generate_schema_pot(w, vreg, schema, libschema=None, cube=None):
    from mx.DateTime import now
    from cubicweb.common.i18n import add_msg
    w('# schema pot file, generated on %s\n' % now().strftime('%Y-%m-%d %H:%M:%S'))
    w('# \n')
    w('# singular and plural forms for each entity type\n')
    w('\n')
    if libschema is not None:
        entities = [e for e in schema.entities() if not e in libschema]
    else:
        entities = schema.entities()
    done = set()
    for eschema in sorted(entities):
        etype = eschema.type
        add_msg(w, etype)
        add_msg(w, '%s_plural' % etype)
        if not eschema.is_final():
            add_msg(w, 'This %s' % etype)
            add_msg(w, 'New %s' % etype)
            add_msg(w, 'add a %s' % etype)
            add_msg(w, 'remove this %s' % etype)
        if eschema.description and not eschema.description in done:
            done.add(eschema.description)
            add_msg(w, eschema.description)
    w('# subject and object forms for each relation type\n')
    w('# (no object form for final relation types)\n')
    w('\n')
    if libschema is not None:
        relations = [r for r in schema.relations() if not r in libschema]
    else:
        relations = schema.relations()
    for rschema in sorted(set(relations)):
        rtype = rschema.type
        add_msg(w, rtype)
138
        done.add(rtype)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
        if not (schema.rschema(rtype).is_final() or rschema.symetric):
            add_msg(w, '%s_object' % rtype)
        if rschema.description and rschema.description not in done:
            done.add(rschema.description)
            add_msg(w, rschema.description)
    w('# add related box generated message\n')
    w('\n')
    for eschema in schema.entities():
        if eschema.is_final():
            continue
        entity = vreg.etype_class(eschema)(None, None)
        for x, rschemas in (('subject', eschema.subject_relations()),
                            ('object', eschema.object_relations())):
            for rschema in rschemas:
                if rschema.is_final():
                    continue
                for teschema in rschema.targets(eschema, x):
                    if defined_in_library(libschema, eschema, rschema, teschema, x):
                        continue
                    if entity.relation_mode(rschema.type, teschema.type, x) == 'create':
                        if x == 'subject':
                            label = 'add %s %s %s %s' % (eschema, rschema, teschema, x)
                            label2 = "creating %s (%s %%(linkto)s %s %s)" % (teschema, eschema, rschema, teschema)
                        else:
                            label = 'add %s %s %s %s' % (teschema, rschema, eschema, x)
                            label2 = "creating %s (%s %s %s %%(linkto)s)" % (teschema, teschema, rschema, eschema)
                        add_msg(w, label)
                        add_msg(w, label2)
Sylvain Thenault's avatar
Sylvain Thenault committed
167
    cube = (cube and 'cubes.%s.' % cube or 'cubicweb.')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
    done = set()
    for reg, objdict in vreg.items():
        for objects in objdict.values():
            for obj in objects:
                objid = '%s_%s' % (reg, obj.id)
                if objid in done:
                    continue
                if obj.__module__.startswith(cube) and obj.property_defs:
                    add_msg(w, '%s_description' % objid)
                    add_msg(w, objid)
                    done.add(objid)
                    
def defined_in_library(libschema, etype, rtype, tetype, x):
    """return true if the given relation definition exists in cubicweb's library"""
    if libschema is None:
        return False
    if x == 'subject':
        subjtype, objtype = etype, tetype
    else:
        subjtype, objtype = tetype, etype
    try:
        return libschema.rschema(rtype).has_rdef(subjtype, objtype)
    except KeyError:
        return False


194
LANGS = ('en', 'fr', 'es')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
I18NDIR = join(BASEDIR, 'i18n')
DEFAULT_POT_HEAD = r'''msgid ""
msgstr ""
"Project-Id-Version: cubicweb %s\n"
"PO-Revision-Date: 2008-03-28 18:14+0100\n"
"Last-Translator: Logilab Team <contact@logilab.fr>\n"
"Language-Team: fr <contact@logilab.fr>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: cubicweb-devtools\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"

''' % cubicwebversion


class UpdateCubicWebCatalogCommand(Command):
    """Update i18n catalogs for cubicweb library.
    
    It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those
    files to add translations of newly added messages.
    """
    name = 'i18nlibupdate'

    def run(self, args):
        """run the command with its specific arguments"""
        if args:
            raise BadCommandUsage('Too much arguments')
        import shutil
        from tempfile import mktemp
        import yams
        from logilab.common.fileutils import ensure_fs_mode
227
        from logilab.common.shellutils import globfind, find, rm
Adrien Di Mascio's avatar
Adrien Di Mascio committed
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
        from cubicweb.common.i18n import extract_from_tal, execute
        tempdir = mktemp()
        mkdir(tempdir)
        potfiles = [join(I18NDIR, 'entities.pot')]
        print '******** extract schema messages'
        schemapot = join(tempdir, 'schema.pot')
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
        schemapotstream = file(schemapot, 'w')
        generate_schema_pot(schemapotstream.write, cubedir=None)
        schemapotstream.close()
        print '******** extract TAL messages'
        tali18nfile = join(tempdir, 'tali18n.py')
        extract_from_tal(find(join(BASEDIR, 'web'), ('.py', '.pt')), tali18nfile)
        print '******** .pot files generation'
244
        for id, files, lang in [('pycubicweb', get_module_files(BASEDIR) + list(globfind(join(BASEDIR, 'misc', 'migration'), '*.py')), None),
245
                                ('schemadescr', globfind(join(BASEDIR, 'schemas'), '*.py'), None),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
246
247
                                ('yams', get_module_files(yams.__path__[0]), None),
                                ('tal', [tali18nfile], None),
248
                                ('js', globfind(join(BASEDIR, 'web'), 'cub*.js'), 'java'),
Adrien Di Mascio's avatar
Adrien Di Mascio committed
249
                                ]:
250
            cmd = 'xgettext --no-location --omit-header -k_ -o %s %s'
Adrien Di Mascio's avatar
Adrien Di Mascio committed
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
            if lang is not None:
                cmd += ' -L %s' % lang
            potfiles.append(join(tempdir, '%s.pot' % id))
            execute(cmd % (potfiles[-1], ' '.join(files)))
        print '******** merging .pot files'
        cubicwebpot = join(tempdir, 'cubicweb.pot')
        execute('msgcat %s > %s' % (' '.join(potfiles), cubicwebpot))
        print '******** merging main pot file with existing translations'
        chdir(I18NDIR)
        toedit = []
        for lang in LANGS:
            target = '%s.po' % lang
            execute('msgmerge -N --sort-output  %s %s > %snew' % (target, cubicwebpot, target))
            ensure_fs_mode(target)
            shutil.move('%snew' % target, target)
            toedit.append(abspath(target))
        # cleanup
268
        rm(tempdir)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
        # instructions pour la suite
        print '*' * 72
        print 'you can now edit the following files:'
        print '* ' + '\n* '.join(toedit)
        print
        print "then you'll have to update cubes catalogs using the i18nupdate command"


class UpdateTemplateCatalogCommand(Command):
    """Update i18n catalogs for cubes. If no cube is specified, update
    catalogs of all registered cubes.
    """
    name = 'i18nupdate'
    arguments = '[<cube>...]'
    
    def run(self, args):
        """run the command with its specific arguments"""
Sylvain Thenault's avatar
Sylvain Thenault committed
286
        CUBEDIR = DevCubeConfiguration.cubes_dir()
Adrien Di Mascio's avatar
Adrien Di Mascio committed
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
        if args:
            cubes = [join(CUBEDIR, app) for app in args]
        else:
            cubes = [join(CUBEDIR, app) for app in listdir(CUBEDIR)
                         if exists(join(CUBEDIR, app, 'i18n'))]
        update_cubes_catalogs(cubes)

def update_cubes_catalogs(cubes):
    import shutil
    from tempfile import mktemp
    from logilab.common.fileutils import ensure_fs_mode
    from logilab.common.shellutils import find, rm
    from cubicweb.common.i18n import extract_from_tal, execute
    toedit = []
    for cubedir in cubes:
        cube = basename(normpath(cubedir))
        if not isdir(cubedir):
            print 'unknown cube', cube
            continue
        tempdir = mktemp()
        mkdir(tempdir)
        print '*' * 72
        print 'updating %s cube...' % cube
        chdir(cubedir)
        potfiles = [join('i18n', scfile) for scfile in ('entities.pot',)
                    if exists(join('i18n', scfile))]
        print '******** extract schema messages'
        schemapot = join(tempdir, 'schema.pot')
        potfiles.append(schemapot)
        # explicit close necessary else the file may not be yet flushed when
        # we'll using it below
        schemapotstream = file(schemapot, 'w')
        generate_schema_pot(schemapotstream.write, cubedir)
        schemapotstream.close()
        print '******** extract TAL messages'
        tali18nfile = join(tempdir, 'tali18n.py')
        extract_from_tal(find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST+('test',)), tali18nfile)
        print '******** extract Javascript messages'
325
        jsfiles =  [jsfile for jsfile in find('.', '.js') if basename(jsfile).startswith('cub')]
Adrien Di Mascio's avatar
Adrien Di Mascio committed
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
        if jsfiles:
            tmppotfile = join(tempdir, 'js.pot')
            execute('xgettext --no-location --omit-header -k_ -L java --from-code=utf-8 -o %s %s'
                    % (tmppotfile, ' '.join(jsfiles)))
            # no pot file created if there are no string to translate
            if exists(tmppotfile): 
                potfiles.append(tmppotfile)
        print '******** create cube specific catalog'
        tmppotfile = join(tempdir, 'generated.pot')
        cubefiles = find('.', '.py', blacklist=STD_BLACKLIST+('test',))
        cubefiles.append(tali18nfile)
        execute('xgettext --no-location --omit-header -k_ -o %s %s'
                % (tmppotfile, ' '.join(cubefiles)))
        if exists(tmppotfile): # doesn't exists of no translation string found
            potfiles.append(tmppotfile)
        potfile = join(tempdir, 'cube.pot')
        print '******** merging .pot files'
        execute('msgcat %s > %s' % (' '.join(potfiles), potfile))
        print '******** merging main pot file with existing translations'
        chdir('i18n')
        for lang in LANGS:
            print '****', lang
            cubepo = '%s.po' % lang
            if not exists(cubepo):
                shutil.copy(potfile, cubepo)
            else:
                execute('msgmerge -N -s %s %s > %snew' % (cubepo, potfile, cubepo))
                ensure_fs_mode(cubepo)
                shutil.move('%snew' % cubepo, cubepo)
            toedit.append(abspath(cubepo))
        # cleanup
        rm(tempdir)
    # instructions pour la suite
    print '*' * 72
    print 'you can now edit the following files:'
    print '* ' + '\n* '.join(toedit)


class LiveServerCommand(Command):
    """Run a server from within a cube directory.
    """
    name = 'live-server'
    arguments = ''
    options = ()
    
    def run(self, args):
        """run the command with its specific arguments"""
        from cubicweb.devtools.livetest import runserver
        runserver()


377
class NewCubeCommand(Command):
Adrien Di Mascio's avatar
Adrien Di Mascio committed
378
379
380
381
382
383
384
385
    """Create a new cube.

    <cubename>
      the name of the new cube
    """
    name = 'newcube'
    arguments = '<cubename>'

386
387
388
389
390
391
392
    options = (
        ("verbose",
         {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',
          'default': 'n',
          'help': 'verbose mode: will ask all possible configuration questions',
          }
         ),
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
        ("author",
         {'short': 'a', 'type' : 'string', 'metavar': '<author>',
          'default': 'LOGILAB S.A. (Paris, FRANCE)',
          'help': 'cube author',
          }
         ),
        ("author-email",
         {'short': 'e', 'type' : 'string', 'metavar': '<email>',
          'default': 'contact@logilab.fr',
          'help': 'cube author\'s email',
          }
         ),
        ("author-web-site",
         {'short': 'w', 'type' : 'string', 'metavar': '<web site>',
          'default': 'http://www.logilab.fr',
          'help': 'cube author\'s web site',
          }
         ),
411
412
        )

Adrien Di Mascio's avatar
Adrien Di Mascio committed
413
414
415
416
417
    
    def run(self, args):
        if len(args) != 1:
            raise BadCommandUsage("exactly one argument (cube name) is expected")
        cubename, = args
418
419
        #if ServerConfiguration.mode != "dev":
        #    self.fail("you can only create new cubes in development mode")
420
        verbose = self.get('verbose')
Adrien Di Mascio's avatar
Adrien Di Mascio committed
421
422
423
424
425
426
427
428
429
430
431
        cubedir = ServerConfiguration.CUBES_DIR
        if not isdir(cubedir):
            print "creating apps directory", cubedir
            try:
                mkdir(cubedir)
            except OSError, err:
                self.fail("failed to create directory %r\n(%s)" % (cubedir, err))
        cubedir = join(cubedir, cubename)
        if exists(cubedir):
            self.fail("%s already exists !" % (cubedir))
        skeldir = join(BASEDIR, 'skeleton')
432
433
434
435
436
437
438
439
        if verbose:
            distname = raw_input('Debian name for your cube (just type enter to use the cube name): ').strip()
            if not distname:
                distname = 'cubicweb-%s' % cubename.lower()
            elif not distname.startswith('cubicweb-'):
                if confirm('do you mean cubicweb-%s ?' % distname):
                    distname = 'cubicweb-' + distname
        else:
Adrien Di Mascio's avatar
Adrien Di Mascio committed
440
            distname = 'cubicweb-%s' % cubename.lower()
441
442
443
444
445
446
447
448
449
450
        
        longdesc = shortdesc = raw_input('Enter a short description for your cube: ')
        if verbose:
            longdesc = raw_input('Enter a long description (or nothing if you want to reuse the short one): ')
        if verbose:
            includes = self._ask_for_dependancies()
            if len(includes) == 1:
                dependancies = '%r,' % includes[0]
            else:
                dependancies = ', '.join(repr(cube) for cube in includes)
Adrien Di Mascio's avatar
Adrien Di Mascio committed
451
        else:
452
            dependancies = ''
Adrien Di Mascio's avatar
Adrien Di Mascio committed
453
454
455
456
457
458
459
460
        from mx.DateTime import now
        context = {'cubename' : cubename,
                   'distname' : distname,
                   'shortdesc' : shortdesc,
                   'longdesc' : longdesc or shortdesc,
                   'dependancies' : dependancies,
                   'version'  : cubicwebversion,
                   'year'  : str(now().year),
461
462
463
                   'author': self['author'],
                   'author-email': self['author-email'],
                   'author-web-site': self['author-web-site'],
Adrien Di Mascio's avatar
Adrien Di Mascio committed
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
                   }
        copy_skeleton(skeldir, cubedir, context)

    def _ask_for_dependancies(self):
        includes = []
        for stdtype in ServerConfiguration.available_cubes():
            ans = raw_input("Depends on cube %s? (N/y/s(kip)/t(ype)"
                            % stdtype).lower().strip()
            if ans == 'y':
                includes.append(stdtype)
            if ans == 't':
                includes = get_csv(raw_input('type dependancies: '))
                break
            elif ans == 's':
                break
        return includes
    
        
register_commands((UpdateCubicWebCatalogCommand,
                   UpdateTemplateCatalogCommand,
                   LiveServerCommand,
485
                   NewCubeCommand,
Adrien Di Mascio's avatar
Adrien Di Mascio committed
486
                   ))