Commit 794ab5db authored by Adrien Di Mascio's avatar Adrien Di Mascio
Browse files

refactor i18n messages extraction

This refactoring will ease later implementation of
i18n cube customization.

related to #15613724
parent 904ee9cd0cf9
......@@ -20,11 +20,12 @@ cubicweb's cubes development
"""
from __future__ import print_function
__docformat__ = "restructuredtext en"
# *ctl module should limit the number of import to be imported as quickly as
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
import shutil
import tempfile
import sys
from datetime import datetime, date
from os import mkdir, chdir, path as osp
......@@ -34,9 +35,12 @@ from pytz import UTC
from six.moves import input
from logilab.common import STD_BLACKLIST
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find
from cubicweb.__pkginfo__ import version as cubicwebversion
from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError
from cubicweb.i18n import extract_from_tal, execute2
from cubicweb.cwctl import CWCTL
from cubicweb.cwconfig import CubicWebNoAppConfiguration
from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,
......@@ -45,6 +49,9 @@ from cubicweb.web.webconfig import WebConfiguration
from cubicweb.server.serverconfig import ServerConfiguration
__docformat__ = "restructuredtext en"
STD_BLACKLIST = set(STD_BLACKLIST)
STD_BLACKLIST.add('.tox')
STD_BLACKLIST.add('test')
......@@ -432,66 +439,140 @@ def update_cubes_catalogs(cubes):
'<yourinstance>" to see changes in your instances.')
return True
def update_cube_catalogs(cubedir):
import shutil
import tempfile
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find, rm
from cubicweb.i18n import extract_from_tal, execute2
cube = osp.basename(osp.normpath(cubedir))
tempdir = tempfile.mkdtemp()
print(underline_title('Updating i18n catalogs for cube %s' % cube))
chdir(cubedir)
if osp.exists(osp.join('i18n', 'entities.pot')):
warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'
% osp.join('i18n', 'entities.pot'), DeprecationWarning)
potfiles = [osp.join('i18n', 'entities.pot')]
elif osp.exists(osp.join('i18n', 'static-messages.pot')):
potfiles = [osp.join('i18n', 'static-messages.pot')]
else:
class I18nCubeMessageExtractor(object):
"""This class encapsulates all the xgettext extraction logic
``generate_pot_file`` is the main entry point called by the ``i18ncube``
command. A cube might decide to customize extractors to ignore a given
directory or to extract messages from a new file type (e.g. .jinja2 files)
For each file type, the class must define two methods:
- ``collect_{filetype}()`` that must return the list of files
xgettext should inspect,
- ``extract_{filetype}(files)`` that calls xgettext and returns the
path to the generated ``pot`` file
"""
blacklist = STD_BLACKLIST
formats = ['tal', 'js', 'py']
def __init__(self, workdir, cubedir):
self.workdir = workdir
self.cubedir = cubedir
def generate_pot_file(self):
"""main entry point: return the generated ``cube.pot`` file
This function first generates all the pot files (schema, tal,
py, js) and then merges them in a single ``cube.pot`` that will
be used to eventually update the ``i18n/*.po`` files.
"""
potfiles = self.generate_pot_files()
potfile = osp.join(self.workdir, 'cube.pot')
print('-> merging %i .pot files' % len(potfiles))
cmd = ['msgcat', '-o', potfile]
cmd.extend(potfiles)
execute2(cmd)
return potfile if osp.exists(potfile) else None
def find(self, exts, blacklist=None):
"""collect files with extensions ``exts`` in the cube directory
"""
if blacklist is None:
blacklist = self.blacklist
return find(self.cubedir, exts, blacklist=blacklist)
def generate_pot_files(self):
"""generate and return the list of all ``pot`` files for the cube
- static-messages.pot,
- schema.pot,
- one ``pot`` file for each inspected format (.py, .js, etc.)
"""
print('-> extracting messages:', end=' ')
potfiles = []
print('-> extracting messages:', end=' ')
print('schema', end=' ')
schemapot = osp.join(tempdir, 'schema.pot')
potfiles.append(schemapot)
# explicit close necessary else the file may not be yet flushed when
# we'll using it below
schemapotstream = open(schemapot, 'w')
generate_schema_pot(schemapotstream.write, cubedir)
schemapotstream.close()
print('TAL', end=' ')
tali18nfile = osp.join(tempdir, 'tali18n.py')
ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST)
extract_from_tal(ptfiles, tali18nfile)
print('Javascript')
jsfiles = [jsfile for jsfile in find('.', '.js')
if osp.basename(jsfile).startswith('cub')]
if jsfiles:
tmppotfile = osp.join(tempdir, 'js.pot')
cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-L', 'java',
'--from-code=utf-8', '-o', tmppotfile] + jsfiles
# static messages
if osp.exists(osp.join('i18n', 'entities.pot')):
warn('entities.pot is deprecated, rename file '
'to static-messages.pot (%s)'
% osp.join('i18n', 'entities.pot'), DeprecationWarning)
potfiles.append(osp.join('i18n', 'entities.pot'))
elif osp.exists(osp.join('i18n', 'static-messages.pot')):
potfiles.append(osp.join('i18n', 'static-messages.pot'))
# messages from schema
potfiles.append(self.schemapot())
# messages from sourcecode
for fmt in self.formats:
collector = getattr(self, 'collect_{0}'.format(fmt))
extractor = getattr(self, 'extract_{0}'.format(fmt))
files = collector()
if files:
potfile = extractor(files)
if potfile:
potfiles.append(potfile)
return potfiles
def schemapot(self):
"""generate the ``schema.pot`` file"""
schemapot = osp.join(self.workdir, 'schema.pot')
print('schema', end=' ')
# explicit close necessary else the file may not be yet flushed when
# we'll using it below
schemapotstream = open(schemapot, 'w')
generate_schema_pot(schemapotstream.write, self.cubedir)
schemapotstream.close()
return schemapot
def _xgettext(self, files, output, k='_', extraopts=''):
"""shortcut to execute the xgettext command and return output file
"""
tmppotfile = osp.join(self.workdir, output)
cmd = ['xgettext', '--no-location', '--omit-header', '-k' + k,
'-o', tmppotfile] + extraopts.split() + files
execute2(cmd)
# no pot file created if there are no string to translate
if osp.exists(tmppotfile):
potfiles.append(tmppotfile)
print('-> creating cube-specific catalog')
tmppotfile = osp.join(tempdir, 'generated.pot')
cubefiles = find('.', '.py', blacklist=STD_BLACKLIST)
cubefiles.append(tali18nfile)
cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-o', tmppotfile]
cmd.extend(cubefiles)
execute2(cmd)
if osp.exists(tmppotfile): # doesn't exists of no translation string found
potfiles.append(tmppotfile)
potfile = osp.join(tempdir, 'cube.pot')
print('-> merging %i .pot files' % len(potfiles))
cmd = ['msgcat', '-o', potfile]
cmd.extend(potfiles)
execute2(cmd)
if not osp.exists(potfile):
return tmppotfile
def collect_tal(self):
print('TAL', end=' ')
return self.find(('.py', '.pt'))
def extract_tal(self, files):
tali18nfile = osp.join(self.workdir, 'tali18n.py')
extract_from_tal(files, tali18nfile)
return self._xgettext(files, output='tal.pot')
def collect_js(self):
print('Javascript')
return [jsfile for jsfile in self.find('.js')
if osp.basename(jsfile).startswith('cub')]
def extract_js(self, files):
return self._xgettext(files, output='js.pot',
extraopts='-L java --from-code=utf-8')
def collect_py(self):
print('-> creating cube-specific catalog')
return self.find('.py')
def extract_py(self, files):
return self._xgettext(files, output='py.pot')
def update_cube_catalogs(cubedir):
cubedir = osp.abspath(osp.normpath(cubedir))
workdir = tempfile.mkdtemp()
cube = osp.basename(cubedir)
print('cubedir', cubedir)
print(underline_title('Updating i18n catalogs for cube %s' % cube))
chdir(cubedir)
extractor = I18nCubeMessageExtractor(workdir, cubedir)
potfile = extractor.generate_pot_file()
if potfile is None:
print('no message catalog for cube', cube, 'nothing to translate')
# cleanup
rm(tempdir)
shutil.rmtree(workdir)
return ()
print('-> merging main pot file with existing translations:', end=' ')
chdir('i18n')
......@@ -502,14 +583,15 @@ def update_cube_catalogs(cubedir):
if not osp.exists(cubepo):
shutil.copy(potfile, cubepo)
else:
cmd = ['msgmerge','-N','-s','-o', cubepo+'new', cubepo, potfile]
cmd = ['msgmerge', '-N', '-s', '-o', cubepo + 'new',
cubepo, potfile]
execute2(cmd)
ensure_fs_mode(cubepo)
shutil.move('%snew' % cubepo, cubepo)
toedit.append(osp.abspath(cubepo))
print()
# cleanup
rm(tempdir)
shutil.rmtree(workdir)
return toedit
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment