1042 lines
38 KiB
Python
1042 lines
38 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
"""
|
||
|
|
babel.messages.frontend
|
||
|
|
~~~~~~~~~~~~~~~~~~~~~~~
|
||
|
|
|
||
|
|
Frontends for the message extraction functionality.
|
||
|
|
|
||
|
|
:copyright: (c) 2013-2021 by the Babel Team.
|
||
|
|
:license: BSD, see LICENSE for more details.
|
||
|
|
"""
|
||
|
|
from __future__ import print_function
|
||
|
|
|
||
|
|
import logging
|
||
|
|
import optparse
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import shutil
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
from collections import OrderedDict
|
||
|
|
from datetime import datetime
|
||
|
|
from locale import getpreferredencoding
|
||
|
|
|
||
|
|
from babel import __version__ as VERSION
|
||
|
|
from babel import Locale, localedata
|
||
|
|
from babel._compat import StringIO, string_types, text_type, PY2
|
||
|
|
from babel.core import UnknownLocaleError
|
||
|
|
from babel.messages.catalog import Catalog
|
||
|
|
from babel.messages.extract import DEFAULT_KEYWORDS, DEFAULT_MAPPING, check_and_call_extract_file, extract_from_dir
|
||
|
|
from babel.messages.mofile import write_mo
|
||
|
|
from babel.messages.pofile import read_po, write_po
|
||
|
|
from babel.util import LOCALTZ
|
||
|
|
from distutils import log as distutils_log
|
||
|
|
from distutils.cmd import Command as _Command
|
||
|
|
from distutils.errors import DistutilsOptionError, DistutilsSetupError
|
||
|
|
|
||
|
|
try:
|
||
|
|
from ConfigParser import RawConfigParser
|
||
|
|
except ImportError:
|
||
|
|
from configparser import RawConfigParser
|
||
|
|
|
||
|
|
|
||
|
|
po_file_read_mode = ('rU' if PY2 else 'r')
|
||
|
|
|
||
|
|
|
||
|
|
def listify_value(arg, split=None):
|
||
|
|
"""
|
||
|
|
Make a list out of an argument.
|
||
|
|
|
||
|
|
Values from `distutils` argument parsing are always single strings;
|
||
|
|
values from `optparse` parsing may be lists of strings that may need
|
||
|
|
to be further split.
|
||
|
|
|
||
|
|
No matter the input, this function returns a flat list of whitespace-trimmed
|
||
|
|
strings, with `None` values filtered out.
|
||
|
|
|
||
|
|
>>> listify_value("foo bar")
|
||
|
|
['foo', 'bar']
|
||
|
|
>>> listify_value(["foo bar"])
|
||
|
|
['foo', 'bar']
|
||
|
|
>>> listify_value([["foo"], "bar"])
|
||
|
|
['foo', 'bar']
|
||
|
|
>>> listify_value([["foo"], ["bar", None, "foo"]])
|
||
|
|
['foo', 'bar', 'foo']
|
||
|
|
>>> listify_value("foo, bar, quux", ",")
|
||
|
|
['foo', 'bar', 'quux']
|
||
|
|
|
||
|
|
:param arg: A string or a list of strings
|
||
|
|
:param split: The argument to pass to `str.split()`.
|
||
|
|
:return:
|
||
|
|
"""
|
||
|
|
out = []
|
||
|
|
|
||
|
|
if not isinstance(arg, (list, tuple)):
|
||
|
|
arg = [arg]
|
||
|
|
|
||
|
|
for val in arg:
|
||
|
|
if val is None:
|
||
|
|
continue
|
||
|
|
if isinstance(val, (list, tuple)):
|
||
|
|
out.extend(listify_value(val, split=split))
|
||
|
|
continue
|
||
|
|
out.extend(s.strip() for s in text_type(val).split(split))
|
||
|
|
assert all(isinstance(val, string_types) for val in out)
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
class Command(_Command):
|
||
|
|
# This class is a small shim between Distutils commands and
|
||
|
|
# optparse option parsing in the frontend command line.
|
||
|
|
|
||
|
|
#: Option name to be input as `args` on the script command line.
|
||
|
|
as_args = None
|
||
|
|
|
||
|
|
#: Options which allow multiple values.
|
||
|
|
#: This is used by the `optparse` transmogrification code.
|
||
|
|
multiple_value_options = ()
|
||
|
|
|
||
|
|
#: Options which are booleans.
|
||
|
|
#: This is used by the `optparse` transmogrification code.
|
||
|
|
# (This is actually used by distutils code too, but is never
|
||
|
|
# declared in the base class.)
|
||
|
|
boolean_options = ()
|
||
|
|
|
||
|
|
#: Option aliases, to retain standalone command compatibility.
|
||
|
|
#: Distutils does not support option aliases, but optparse does.
|
||
|
|
#: This maps the distutils argument name to an iterable of aliases
|
||
|
|
#: that are usable with optparse.
|
||
|
|
option_aliases = {}
|
||
|
|
|
||
|
|
#: Choices for options that needed to be restricted to specific
|
||
|
|
#: list of choices.
|
||
|
|
option_choices = {}
|
||
|
|
|
||
|
|
#: Log object. To allow replacement in the script command line runner.
|
||
|
|
log = distutils_log
|
||
|
|
|
||
|
|
def __init__(self, dist=None):
|
||
|
|
# A less strict version of distutils' `__init__`.
|
||
|
|
self.distribution = dist
|
||
|
|
self.initialize_options()
|
||
|
|
self._dry_run = None
|
||
|
|
self.verbose = False
|
||
|
|
self.force = None
|
||
|
|
self.help = 0
|
||
|
|
self.finalized = 0
|
||
|
|
|
||
|
|
|
||
|
|
class compile_catalog(Command):
|
||
|
|
"""Catalog compilation command for use in ``setup.py`` scripts.
|
||
|
|
|
||
|
|
If correctly installed, this command is available to Setuptools-using
|
||
|
|
setup scripts automatically. For projects using plain old ``distutils``,
|
||
|
|
the command needs to be registered explicitly in ``setup.py``::
|
||
|
|
|
||
|
|
from babel.messages.frontend import compile_catalog
|
||
|
|
|
||
|
|
setup(
|
||
|
|
...
|
||
|
|
cmdclass = {'compile_catalog': compile_catalog}
|
||
|
|
)
|
||
|
|
|
||
|
|
.. versionadded:: 0.9
|
||
|
|
"""
|
||
|
|
|
||
|
|
description = 'compile message catalogs to binary MO files'
|
||
|
|
user_options = [
|
||
|
|
('domain=', 'D',
|
||
|
|
"domains of PO files (space separated list, default 'messages')"),
|
||
|
|
('directory=', 'd',
|
||
|
|
'path to base directory containing the catalogs'),
|
||
|
|
('input-file=', 'i',
|
||
|
|
'name of the input file'),
|
||
|
|
('output-file=', 'o',
|
||
|
|
"name of the output file (default "
|
||
|
|
"'<output_dir>/<locale>/LC_MESSAGES/<domain>.mo')"),
|
||
|
|
('locale=', 'l',
|
||
|
|
'locale of the catalog to compile'),
|
||
|
|
('use-fuzzy', 'f',
|
||
|
|
'also include fuzzy translations'),
|
||
|
|
('statistics', None,
|
||
|
|
'print statistics about translations')
|
||
|
|
]
|
||
|
|
boolean_options = ['use-fuzzy', 'statistics']
|
||
|
|
|
||
|
|
def initialize_options(self):
|
||
|
|
self.domain = 'messages'
|
||
|
|
self.directory = None
|
||
|
|
self.input_file = None
|
||
|
|
self.output_file = None
|
||
|
|
self.locale = None
|
||
|
|
self.use_fuzzy = False
|
||
|
|
self.statistics = False
|
||
|
|
|
||
|
|
def finalize_options(self):
|
||
|
|
self.domain = listify_value(self.domain)
|
||
|
|
if not self.input_file and not self.directory:
|
||
|
|
raise DistutilsOptionError('you must specify either the input file '
|
||
|
|
'or the base directory')
|
||
|
|
if not self.output_file and not self.directory:
|
||
|
|
raise DistutilsOptionError('you must specify either the output file '
|
||
|
|
'or the base directory')
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
n_errors = 0
|
||
|
|
for domain in self.domain:
|
||
|
|
for catalog, errors in self._run_domain(domain).items():
|
||
|
|
n_errors += len(errors)
|
||
|
|
if n_errors:
|
||
|
|
self.log.error('%d errors encountered.' % n_errors)
|
||
|
|
return (1 if n_errors else 0)
|
||
|
|
|
||
|
|
def _run_domain(self, domain):
|
||
|
|
po_files = []
|
||
|
|
mo_files = []
|
||
|
|
|
||
|
|
if not self.input_file:
|
||
|
|
if self.locale:
|
||
|
|
po_files.append((self.locale,
|
||
|
|
os.path.join(self.directory, self.locale,
|
||
|
|
'LC_MESSAGES',
|
||
|
|
domain + '.po')))
|
||
|
|
mo_files.append(os.path.join(self.directory, self.locale,
|
||
|
|
'LC_MESSAGES',
|
||
|
|
domain + '.mo'))
|
||
|
|
else:
|
||
|
|
for locale in os.listdir(self.directory):
|
||
|
|
po_file = os.path.join(self.directory, locale,
|
||
|
|
'LC_MESSAGES', domain + '.po')
|
||
|
|
if os.path.exists(po_file):
|
||
|
|
po_files.append((locale, po_file))
|
||
|
|
mo_files.append(os.path.join(self.directory, locale,
|
||
|
|
'LC_MESSAGES',
|
||
|
|
domain + '.mo'))
|
||
|
|
else:
|
||
|
|
po_files.append((self.locale, self.input_file))
|
||
|
|
if self.output_file:
|
||
|
|
mo_files.append(self.output_file)
|
||
|
|
else:
|
||
|
|
mo_files.append(os.path.join(self.directory, self.locale,
|
||
|
|
'LC_MESSAGES',
|
||
|
|
domain + '.mo'))
|
||
|
|
|
||
|
|
if not po_files:
|
||
|
|
raise DistutilsOptionError('no message catalogs found')
|
||
|
|
|
||
|
|
catalogs_and_errors = {}
|
||
|
|
|
||
|
|
for idx, (locale, po_file) in enumerate(po_files):
|
||
|
|
mo_file = mo_files[idx]
|
||
|
|
with open(po_file, 'rb') as infile:
|
||
|
|
catalog = read_po(infile, locale)
|
||
|
|
|
||
|
|
if self.statistics:
|
||
|
|
translated = 0
|
||
|
|
for message in list(catalog)[1:]:
|
||
|
|
if message.string:
|
||
|
|
translated += 1
|
||
|
|
percentage = 0
|
||
|
|
if len(catalog):
|
||
|
|
percentage = translated * 100 // len(catalog)
|
||
|
|
self.log.info(
|
||
|
|
'%d of %d messages (%d%%) translated in %s',
|
||
|
|
translated, len(catalog), percentage, po_file
|
||
|
|
)
|
||
|
|
|
||
|
|
if catalog.fuzzy and not self.use_fuzzy:
|
||
|
|
self.log.info('catalog %s is marked as fuzzy, skipping', po_file)
|
||
|
|
continue
|
||
|
|
|
||
|
|
catalogs_and_errors[catalog] = catalog_errors = list(catalog.check())
|
||
|
|
for message, errors in catalog_errors:
|
||
|
|
for error in errors:
|
||
|
|
self.log.error(
|
||
|
|
'error: %s:%d: %s', po_file, message.lineno, error
|
||
|
|
)
|
||
|
|
|
||
|
|
self.log.info('compiling catalog %s to %s', po_file, mo_file)
|
||
|
|
|
||
|
|
with open(mo_file, 'wb') as outfile:
|
||
|
|
write_mo(outfile, catalog, use_fuzzy=self.use_fuzzy)
|
||
|
|
|
||
|
|
return catalogs_and_errors
|
||
|
|
|
||
|
|
|
||
|
|
class extract_messages(Command):
|
||
|
|
"""Message extraction command for use in ``setup.py`` scripts.
|
||
|
|
|
||
|
|
If correctly installed, this command is available to Setuptools-using
|
||
|
|
setup scripts automatically. For projects using plain old ``distutils``,
|
||
|
|
the command needs to be registered explicitly in ``setup.py``::
|
||
|
|
|
||
|
|
from babel.messages.frontend import extract_messages
|
||
|
|
|
||
|
|
setup(
|
||
|
|
...
|
||
|
|
cmdclass = {'extract_messages': extract_messages}
|
||
|
|
)
|
||
|
|
"""
|
||
|
|
|
||
|
|
description = 'extract localizable strings from the project code'
|
||
|
|
user_options = [
|
||
|
|
('charset=', None,
|
||
|
|
'charset to use in the output file (default "utf-8")'),
|
||
|
|
('keywords=', 'k',
|
||
|
|
'space-separated list of keywords to look for in addition to the '
|
||
|
|
'defaults (may be repeated multiple times)'),
|
||
|
|
('no-default-keywords', None,
|
||
|
|
'do not include the default keywords'),
|
||
|
|
('mapping-file=', 'F',
|
||
|
|
'path to the mapping configuration file'),
|
||
|
|
('no-location', None,
|
||
|
|
'do not include location comments with filename and line number'),
|
||
|
|
('add-location=', None,
|
||
|
|
'location lines format. If it is not given or "full", it generates '
|
||
|
|
'the lines with both file name and line number. If it is "file", '
|
||
|
|
'the line number part is omitted. If it is "never", it completely '
|
||
|
|
'suppresses the lines (same as --no-location).'),
|
||
|
|
('omit-header', None,
|
||
|
|
'do not include msgid "" entry in header'),
|
||
|
|
('output-file=', 'o',
|
||
|
|
'name of the output file'),
|
||
|
|
('width=', 'w',
|
||
|
|
'set output line width (default 76)'),
|
||
|
|
('no-wrap', None,
|
||
|
|
'do not break long message lines, longer than the output line width, '
|
||
|
|
'into several lines'),
|
||
|
|
('sort-output', None,
|
||
|
|
'generate sorted output (default False)'),
|
||
|
|
('sort-by-file', None,
|
||
|
|
'sort output by file location (default False)'),
|
||
|
|
('msgid-bugs-address=', None,
|
||
|
|
'set report address for msgid'),
|
||
|
|
('copyright-holder=', None,
|
||
|
|
'set copyright holder in output'),
|
||
|
|
('project=', None,
|
||
|
|
'set project name in output'),
|
||
|
|
('version=', None,
|
||
|
|
'set project version in output'),
|
||
|
|
('add-comments=', 'c',
|
||
|
|
'place comment block with TAG (or those preceding keyword lines) in '
|
||
|
|
'output file. Separate multiple TAGs with commas(,)'), # TODO: Support repetition of this argument
|
||
|
|
('strip-comments', 's',
|
||
|
|
'strip the comment TAGs from the comments.'),
|
||
|
|
('input-paths=', None,
|
||
|
|
'files or directories that should be scanned for messages. Separate multiple '
|
||
|
|
'files or directories with commas(,)'), # TODO: Support repetition of this argument
|
||
|
|
('input-dirs=', None, # TODO (3.x): Remove me.
|
||
|
|
'alias for input-paths (does allow files as well as directories).'),
|
||
|
|
]
|
||
|
|
boolean_options = [
|
||
|
|
'no-default-keywords', 'no-location', 'omit-header', 'no-wrap',
|
||
|
|
'sort-output', 'sort-by-file', 'strip-comments'
|
||
|
|
]
|
||
|
|
as_args = 'input-paths'
|
||
|
|
multiple_value_options = ('add-comments', 'keywords')
|
||
|
|
option_aliases = {
|
||
|
|
'keywords': ('--keyword',),
|
||
|
|
'mapping-file': ('--mapping',),
|
||
|
|
'output-file': ('--output',),
|
||
|
|
'strip-comments': ('--strip-comment-tags',),
|
||
|
|
}
|
||
|
|
option_choices = {
|
||
|
|
'add-location': ('full', 'file', 'never',),
|
||
|
|
}
|
||
|
|
|
||
|
|
def initialize_options(self):
|
||
|
|
self.charset = 'utf-8'
|
||
|
|
self.keywords = None
|
||
|
|
self.no_default_keywords = False
|
||
|
|
self.mapping_file = None
|
||
|
|
self.no_location = False
|
||
|
|
self.add_location = None
|
||
|
|
self.omit_header = False
|
||
|
|
self.output_file = None
|
||
|
|
self.input_dirs = None
|
||
|
|
self.input_paths = None
|
||
|
|
self.width = None
|
||
|
|
self.no_wrap = False
|
||
|
|
self.sort_output = False
|
||
|
|
self.sort_by_file = False
|
||
|
|
self.msgid_bugs_address = None
|
||
|
|
self.copyright_holder = None
|
||
|
|
self.project = None
|
||
|
|
self.version = None
|
||
|
|
self.add_comments = None
|
||
|
|
self.strip_comments = False
|
||
|
|
self.include_lineno = True
|
||
|
|
|
||
|
|
def finalize_options(self):
|
||
|
|
if self.input_dirs:
|
||
|
|
if not self.input_paths:
|
||
|
|
self.input_paths = self.input_dirs
|
||
|
|
else:
|
||
|
|
raise DistutilsOptionError(
|
||
|
|
'input-dirs and input-paths are mutually exclusive'
|
||
|
|
)
|
||
|
|
|
||
|
|
if self.no_default_keywords:
|
||
|
|
keywords = {}
|
||
|
|
else:
|
||
|
|
keywords = DEFAULT_KEYWORDS.copy()
|
||
|
|
|
||
|
|
keywords.update(parse_keywords(listify_value(self.keywords)))
|
||
|
|
|
||
|
|
self.keywords = keywords
|
||
|
|
|
||
|
|
if not self.keywords:
|
||
|
|
raise DistutilsOptionError('you must specify new keywords if you '
|
||
|
|
'disable the default ones')
|
||
|
|
|
||
|
|
if not self.output_file:
|
||
|
|
raise DistutilsOptionError('no output file specified')
|
||
|
|
if self.no_wrap and self.width:
|
||
|
|
raise DistutilsOptionError("'--no-wrap' and '--width' are mutually "
|
||
|
|
"exclusive")
|
||
|
|
if not self.no_wrap and not self.width:
|
||
|
|
self.width = 76
|
||
|
|
elif self.width is not None:
|
||
|
|
self.width = int(self.width)
|
||
|
|
|
||
|
|
if self.sort_output and self.sort_by_file:
|
||
|
|
raise DistutilsOptionError("'--sort-output' and '--sort-by-file' "
|
||
|
|
"are mutually exclusive")
|
||
|
|
|
||
|
|
if self.input_paths:
|
||
|
|
if isinstance(self.input_paths, string_types):
|
||
|
|
self.input_paths = re.split(r',\s*', self.input_paths)
|
||
|
|
elif self.distribution is not None:
|
||
|
|
self.input_paths = dict.fromkeys([
|
||
|
|
k.split('.', 1)[0]
|
||
|
|
for k in (self.distribution.packages or ())
|
||
|
|
]).keys()
|
||
|
|
else:
|
||
|
|
self.input_paths = []
|
||
|
|
|
||
|
|
if not self.input_paths:
|
||
|
|
raise DistutilsOptionError("no input files or directories specified")
|
||
|
|
|
||
|
|
for path in self.input_paths:
|
||
|
|
if not os.path.exists(path):
|
||
|
|
raise DistutilsOptionError("Input path: %s does not exist" % path)
|
||
|
|
|
||
|
|
self.add_comments = listify_value(self.add_comments or (), ",")
|
||
|
|
|
||
|
|
if self.distribution:
|
||
|
|
if not self.project:
|
||
|
|
self.project = self.distribution.get_name()
|
||
|
|
if not self.version:
|
||
|
|
self.version = self.distribution.get_version()
|
||
|
|
|
||
|
|
if self.add_location == 'never':
|
||
|
|
self.no_location = True
|
||
|
|
elif self.add_location == 'file':
|
||
|
|
self.include_lineno = False
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
mappings = self._get_mappings()
|
||
|
|
with open(self.output_file, 'wb') as outfile:
|
||
|
|
catalog = Catalog(project=self.project,
|
||
|
|
version=self.version,
|
||
|
|
msgid_bugs_address=self.msgid_bugs_address,
|
||
|
|
copyright_holder=self.copyright_holder,
|
||
|
|
charset=self.charset)
|
||
|
|
|
||
|
|
for path, method_map, options_map in mappings:
|
||
|
|
def callback(filename, method, options):
|
||
|
|
if method == 'ignore':
|
||
|
|
return
|
||
|
|
|
||
|
|
# If we explicitly provide a full filepath, just use that.
|
||
|
|
# Otherwise, path will be the directory path and filename
|
||
|
|
# is the relative path from that dir to the file.
|
||
|
|
# So we can join those to get the full filepath.
|
||
|
|
if os.path.isfile(path):
|
||
|
|
filepath = path
|
||
|
|
else:
|
||
|
|
filepath = os.path.normpath(os.path.join(path, filename))
|
||
|
|
|
||
|
|
optstr = ''
|
||
|
|
if options:
|
||
|
|
optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for
|
||
|
|
k, v in options.items()])
|
||
|
|
self.log.info('extracting messages from %s%s', filepath, optstr)
|
||
|
|
|
||
|
|
if os.path.isfile(path):
|
||
|
|
current_dir = os.getcwd()
|
||
|
|
extracted = check_and_call_extract_file(
|
||
|
|
path, method_map, options_map,
|
||
|
|
callback, self.keywords, self.add_comments,
|
||
|
|
self.strip_comments, current_dir
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
extracted = extract_from_dir(
|
||
|
|
path, method_map, options_map,
|
||
|
|
keywords=self.keywords,
|
||
|
|
comment_tags=self.add_comments,
|
||
|
|
callback=callback,
|
||
|
|
strip_comment_tags=self.strip_comments
|
||
|
|
)
|
||
|
|
for filename, lineno, message, comments, context in extracted:
|
||
|
|
if os.path.isfile(path):
|
||
|
|
filepath = filename # already normalized
|
||
|
|
else:
|
||
|
|
filepath = os.path.normpath(os.path.join(path, filename))
|
||
|
|
|
||
|
|
catalog.add(message, None, [(filepath, lineno)],
|
||
|
|
auto_comments=comments, context=context)
|
||
|
|
|
||
|
|
self.log.info('writing PO template file to %s', self.output_file)
|
||
|
|
write_po(outfile, catalog, width=self.width,
|
||
|
|
no_location=self.no_location,
|
||
|
|
omit_header=self.omit_header,
|
||
|
|
sort_output=self.sort_output,
|
||
|
|
sort_by_file=self.sort_by_file,
|
||
|
|
include_lineno=self.include_lineno)
|
||
|
|
|
||
|
|
def _get_mappings(self):
|
||
|
|
mappings = []
|
||
|
|
|
||
|
|
if self.mapping_file:
|
||
|
|
with open(self.mapping_file, po_file_read_mode) as fileobj:
|
||
|
|
method_map, options_map = parse_mapping(fileobj)
|
||
|
|
for path in self.input_paths:
|
||
|
|
mappings.append((path, method_map, options_map))
|
||
|
|
|
||
|
|
elif getattr(self.distribution, 'message_extractors', None):
|
||
|
|
message_extractors = self.distribution.message_extractors
|
||
|
|
for path, mapping in message_extractors.items():
|
||
|
|
if isinstance(mapping, string_types):
|
||
|
|
method_map, options_map = parse_mapping(StringIO(mapping))
|
||
|
|
else:
|
||
|
|
method_map, options_map = [], {}
|
||
|
|
for pattern, method, options in mapping:
|
||
|
|
method_map.append((pattern, method))
|
||
|
|
options_map[pattern] = options or {}
|
||
|
|
mappings.append((path, method_map, options_map))
|
||
|
|
|
||
|
|
else:
|
||
|
|
for path in self.input_paths:
|
||
|
|
mappings.append((path, DEFAULT_MAPPING, {}))
|
||
|
|
|
||
|
|
return mappings
|
||
|
|
|
||
|
|
|
||
|
|
def check_message_extractors(dist, name, value):
|
||
|
|
"""Validate the ``message_extractors`` keyword argument to ``setup()``.
|
||
|
|
|
||
|
|
:param dist: the distutils/setuptools ``Distribution`` object
|
||
|
|
:param name: the name of the keyword argument (should always be
|
||
|
|
"message_extractors")
|
||
|
|
:param value: the value of the keyword argument
|
||
|
|
:raise `DistutilsSetupError`: if the value is not valid
|
||
|
|
"""
|
||
|
|
assert name == 'message_extractors'
|
||
|
|
if not isinstance(value, dict):
|
||
|
|
raise DistutilsSetupError('the value of the "message_extractors" '
|
||
|
|
'parameter must be a dictionary')
|
||
|
|
|
||
|
|
|
||
|
|
class init_catalog(Command):
|
||
|
|
"""New catalog initialization command for use in ``setup.py`` scripts.
|
||
|
|
|
||
|
|
If correctly installed, this command is available to Setuptools-using
|
||
|
|
setup scripts automatically. For projects using plain old ``distutils``,
|
||
|
|
the command needs to be registered explicitly in ``setup.py``::
|
||
|
|
|
||
|
|
from babel.messages.frontend import init_catalog
|
||
|
|
|
||
|
|
setup(
|
||
|
|
...
|
||
|
|
cmdclass = {'init_catalog': init_catalog}
|
||
|
|
)
|
||
|
|
"""
|
||
|
|
|
||
|
|
description = 'create a new catalog based on a POT file'
|
||
|
|
user_options = [
|
||
|
|
('domain=', 'D',
|
||
|
|
"domain of PO file (default 'messages')"),
|
||
|
|
('input-file=', 'i',
|
||
|
|
'name of the input file'),
|
||
|
|
('output-dir=', 'd',
|
||
|
|
'path to output directory'),
|
||
|
|
('output-file=', 'o',
|
||
|
|
"name of the output file (default "
|
||
|
|
"'<output_dir>/<locale>/LC_MESSAGES/<domain>.po')"),
|
||
|
|
('locale=', 'l',
|
||
|
|
'locale for the new localized catalog'),
|
||
|
|
('width=', 'w',
|
||
|
|
'set output line width (default 76)'),
|
||
|
|
('no-wrap', None,
|
||
|
|
'do not break long message lines, longer than the output line width, '
|
||
|
|
'into several lines'),
|
||
|
|
]
|
||
|
|
boolean_options = ['no-wrap']
|
||
|
|
|
||
|
|
def initialize_options(self):
|
||
|
|
self.output_dir = None
|
||
|
|
self.output_file = None
|
||
|
|
self.input_file = None
|
||
|
|
self.locale = None
|
||
|
|
self.domain = 'messages'
|
||
|
|
self.no_wrap = False
|
||
|
|
self.width = None
|
||
|
|
|
||
|
|
def finalize_options(self):
|
||
|
|
if not self.input_file:
|
||
|
|
raise DistutilsOptionError('you must specify the input file')
|
||
|
|
|
||
|
|
if not self.locale:
|
||
|
|
raise DistutilsOptionError('you must provide a locale for the '
|
||
|
|
'new catalog')
|
||
|
|
try:
|
||
|
|
self._locale = Locale.parse(self.locale)
|
||
|
|
except UnknownLocaleError as e:
|
||
|
|
raise DistutilsOptionError(e)
|
||
|
|
|
||
|
|
if not self.output_file and not self.output_dir:
|
||
|
|
raise DistutilsOptionError('you must specify the output directory')
|
||
|
|
if not self.output_file:
|
||
|
|
self.output_file = os.path.join(self.output_dir, self.locale,
|
||
|
|
'LC_MESSAGES', self.domain + '.po')
|
||
|
|
|
||
|
|
if not os.path.exists(os.path.dirname(self.output_file)):
|
||
|
|
os.makedirs(os.path.dirname(self.output_file))
|
||
|
|
if self.no_wrap and self.width:
|
||
|
|
raise DistutilsOptionError("'--no-wrap' and '--width' are mutually "
|
||
|
|
"exclusive")
|
||
|
|
if not self.no_wrap and not self.width:
|
||
|
|
self.width = 76
|
||
|
|
elif self.width is not None:
|
||
|
|
self.width = int(self.width)
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
self.log.info(
|
||
|
|
'creating catalog %s based on %s', self.output_file, self.input_file
|
||
|
|
)
|
||
|
|
|
||
|
|
with open(self.input_file, 'rb') as infile:
|
||
|
|
# Although reading from the catalog template, read_po must be fed
|
||
|
|
# the locale in order to correctly calculate plurals
|
||
|
|
catalog = read_po(infile, locale=self.locale)
|
||
|
|
|
||
|
|
catalog.locale = self._locale
|
||
|
|
catalog.revision_date = datetime.now(LOCALTZ)
|
||
|
|
catalog.fuzzy = False
|
||
|
|
|
||
|
|
with open(self.output_file, 'wb') as outfile:
|
||
|
|
write_po(outfile, catalog, width=self.width)
|
||
|
|
|
||
|
|
|
||
|
|
class update_catalog(Command):
|
||
|
|
"""Catalog merging command for use in ``setup.py`` scripts.
|
||
|
|
|
||
|
|
If correctly installed, this command is available to Setuptools-using
|
||
|
|
setup scripts automatically. For projects using plain old ``distutils``,
|
||
|
|
the command needs to be registered explicitly in ``setup.py``::
|
||
|
|
|
||
|
|
from babel.messages.frontend import update_catalog
|
||
|
|
|
||
|
|
setup(
|
||
|
|
...
|
||
|
|
cmdclass = {'update_catalog': update_catalog}
|
||
|
|
)
|
||
|
|
|
||
|
|
.. versionadded:: 0.9
|
||
|
|
"""
|
||
|
|
|
||
|
|
description = 'update message catalogs from a POT file'
|
||
|
|
user_options = [
|
||
|
|
('domain=', 'D',
|
||
|
|
"domain of PO file (default 'messages')"),
|
||
|
|
('input-file=', 'i',
|
||
|
|
'name of the input file'),
|
||
|
|
('output-dir=', 'd',
|
||
|
|
'path to base directory containing the catalogs'),
|
||
|
|
('output-file=', 'o',
|
||
|
|
"name of the output file (default "
|
||
|
|
"'<output_dir>/<locale>/LC_MESSAGES/<domain>.po')"),
|
||
|
|
('omit-header', None,
|
||
|
|
"do not include msgid "" entry in header"),
|
||
|
|
('locale=', 'l',
|
||
|
|
'locale of the catalog to compile'),
|
||
|
|
('width=', 'w',
|
||
|
|
'set output line width (default 76)'),
|
||
|
|
('no-wrap', None,
|
||
|
|
'do not break long message lines, longer than the output line width, '
|
||
|
|
'into several lines'),
|
||
|
|
('ignore-obsolete=', None,
|
||
|
|
'whether to omit obsolete messages from the output'),
|
||
|
|
('no-fuzzy-matching', 'N',
|
||
|
|
'do not use fuzzy matching'),
|
||
|
|
('update-header-comment', None,
|
||
|
|
'update target header comment'),
|
||
|
|
('previous', None,
|
||
|
|
'keep previous msgids of translated messages'),
|
||
|
|
]
|
||
|
|
boolean_options = [
|
||
|
|
'omit-header', 'no-wrap', 'ignore-obsolete', 'no-fuzzy-matching',
|
||
|
|
'previous', 'update-header-comment',
|
||
|
|
]
|
||
|
|
|
||
|
|
def initialize_options(self):
|
||
|
|
self.domain = 'messages'
|
||
|
|
self.input_file = None
|
||
|
|
self.output_dir = None
|
||
|
|
self.output_file = None
|
||
|
|
self.omit_header = False
|
||
|
|
self.locale = None
|
||
|
|
self.width = None
|
||
|
|
self.no_wrap = False
|
||
|
|
self.ignore_obsolete = False
|
||
|
|
self.no_fuzzy_matching = False
|
||
|
|
self.update_header_comment = False
|
||
|
|
self.previous = False
|
||
|
|
|
||
|
|
def finalize_options(self):
|
||
|
|
if not self.input_file:
|
||
|
|
raise DistutilsOptionError('you must specify the input file')
|
||
|
|
if not self.output_file and not self.output_dir:
|
||
|
|
raise DistutilsOptionError('you must specify the output file or '
|
||
|
|
'directory')
|
||
|
|
if self.output_file and not self.locale:
|
||
|
|
raise DistutilsOptionError('you must specify the locale')
|
||
|
|
if self.no_wrap and self.width:
|
||
|
|
raise DistutilsOptionError("'--no-wrap' and '--width' are mutually "
|
||
|
|
"exclusive")
|
||
|
|
if not self.no_wrap and not self.width:
|
||
|
|
self.width = 76
|
||
|
|
elif self.width is not None:
|
||
|
|
self.width = int(self.width)
|
||
|
|
if self.no_fuzzy_matching and self.previous:
|
||
|
|
self.previous = False
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
po_files = []
|
||
|
|
if not self.output_file:
|
||
|
|
if self.locale:
|
||
|
|
po_files.append((self.locale,
|
||
|
|
os.path.join(self.output_dir, self.locale,
|
||
|
|
'LC_MESSAGES',
|
||
|
|
self.domain + '.po')))
|
||
|
|
else:
|
||
|
|
for locale in os.listdir(self.output_dir):
|
||
|
|
po_file = os.path.join(self.output_dir, locale,
|
||
|
|
'LC_MESSAGES',
|
||
|
|
self.domain + '.po')
|
||
|
|
if os.path.exists(po_file):
|
||
|
|
po_files.append((locale, po_file))
|
||
|
|
else:
|
||
|
|
po_files.append((self.locale, self.output_file))
|
||
|
|
|
||
|
|
if not po_files:
|
||
|
|
raise DistutilsOptionError('no message catalogs found')
|
||
|
|
|
||
|
|
domain = self.domain
|
||
|
|
if not domain:
|
||
|
|
domain = os.path.splitext(os.path.basename(self.input_file))[0]
|
||
|
|
|
||
|
|
with open(self.input_file, 'rb') as infile:
|
||
|
|
template = read_po(infile)
|
||
|
|
|
||
|
|
for locale, filename in po_files:
|
||
|
|
self.log.info('updating catalog %s based on %s', filename, self.input_file)
|
||
|
|
with open(filename, 'rb') as infile:
|
||
|
|
catalog = read_po(infile, locale=locale, domain=domain)
|
||
|
|
|
||
|
|
catalog.update(
|
||
|
|
template, self.no_fuzzy_matching,
|
||
|
|
update_header_comment=self.update_header_comment
|
||
|
|
)
|
||
|
|
|
||
|
|
tmpname = os.path.join(os.path.dirname(filename),
|
||
|
|
tempfile.gettempprefix() +
|
||
|
|
os.path.basename(filename))
|
||
|
|
try:
|
||
|
|
with open(tmpname, 'wb') as tmpfile:
|
||
|
|
write_po(tmpfile, catalog,
|
||
|
|
omit_header=self.omit_header,
|
||
|
|
ignore_obsolete=self.ignore_obsolete,
|
||
|
|
include_previous=self.previous, width=self.width)
|
||
|
|
except:
|
||
|
|
os.remove(tmpname)
|
||
|
|
raise
|
||
|
|
|
||
|
|
try:
|
||
|
|
os.rename(tmpname, filename)
|
||
|
|
except OSError:
|
||
|
|
# We're probably on Windows, which doesn't support atomic
|
||
|
|
# renames, at least not through Python
|
||
|
|
# If the error is in fact due to a permissions problem, that
|
||
|
|
# same error is going to be raised from one of the following
|
||
|
|
# operations
|
||
|
|
os.remove(filename)
|
||
|
|
shutil.copy(tmpname, filename)
|
||
|
|
os.remove(tmpname)
|
||
|
|
|
||
|
|
|
||
|
|
class CommandLineInterface(object):
|
||
|
|
"""Command-line interface.
|
||
|
|
|
||
|
|
This class provides a simple command-line interface to the message
|
||
|
|
extraction and PO file generation functionality.
|
||
|
|
"""
|
||
|
|
|
||
|
|
usage = '%%prog %s [options] %s'
|
||
|
|
version = '%%prog %s' % VERSION
|
||
|
|
commands = {
|
||
|
|
'compile': 'compile message catalogs to MO files',
|
||
|
|
'extract': 'extract messages from source files and generate a POT file',
|
||
|
|
'init': 'create new message catalogs from a POT file',
|
||
|
|
'update': 'update existing message catalogs from a POT file'
|
||
|
|
}
|
||
|
|
|
||
|
|
command_classes = {
|
||
|
|
'compile': compile_catalog,
|
||
|
|
'extract': extract_messages,
|
||
|
|
'init': init_catalog,
|
||
|
|
'update': update_catalog,
|
||
|
|
}
|
||
|
|
|
||
|
|
log = None # Replaced on instance level
|
||
|
|
|
||
|
|
def run(self, argv=None):
|
||
|
|
"""Main entry point of the command-line interface.
|
||
|
|
|
||
|
|
:param argv: list of arguments passed on the command-line
|
||
|
|
"""
|
||
|
|
|
||
|
|
if argv is None:
|
||
|
|
argv = sys.argv
|
||
|
|
|
||
|
|
self.parser = optparse.OptionParser(usage=self.usage % ('command', '[args]'),
|
||
|
|
version=self.version)
|
||
|
|
self.parser.disable_interspersed_args()
|
||
|
|
self.parser.print_help = self._help
|
||
|
|
self.parser.add_option('--list-locales', dest='list_locales',
|
||
|
|
action='store_true',
|
||
|
|
help="print all known locales and exit")
|
||
|
|
self.parser.add_option('-v', '--verbose', action='store_const',
|
||
|
|
dest='loglevel', const=logging.DEBUG,
|
||
|
|
help='print as much as possible')
|
||
|
|
self.parser.add_option('-q', '--quiet', action='store_const',
|
||
|
|
dest='loglevel', const=logging.ERROR,
|
||
|
|
help='print as little as possible')
|
||
|
|
self.parser.set_defaults(list_locales=False, loglevel=logging.INFO)
|
||
|
|
|
||
|
|
options, args = self.parser.parse_args(argv[1:])
|
||
|
|
|
||
|
|
self._configure_logging(options.loglevel)
|
||
|
|
if options.list_locales:
|
||
|
|
identifiers = localedata.locale_identifiers()
|
||
|
|
longest = max([len(identifier) for identifier in identifiers])
|
||
|
|
identifiers.sort()
|
||
|
|
format = u'%%-%ds %%s' % (longest + 1)
|
||
|
|
for identifier in identifiers:
|
||
|
|
locale = Locale.parse(identifier)
|
||
|
|
output = format % (identifier, locale.english_name)
|
||
|
|
print(output.encode(sys.stdout.encoding or
|
||
|
|
getpreferredencoding() or
|
||
|
|
'ascii', 'replace'))
|
||
|
|
return 0
|
||
|
|
|
||
|
|
if not args:
|
||
|
|
self.parser.error('no valid command or option passed. '
|
||
|
|
'Try the -h/--help option for more information.')
|
||
|
|
|
||
|
|
cmdname = args[0]
|
||
|
|
if cmdname not in self.commands:
|
||
|
|
self.parser.error('unknown command "%s"' % cmdname)
|
||
|
|
|
||
|
|
cmdinst = self._configure_command(cmdname, args[1:])
|
||
|
|
return cmdinst.run()
|
||
|
|
|
||
|
|
def _configure_logging(self, loglevel):
|
||
|
|
self.log = logging.getLogger('babel')
|
||
|
|
self.log.setLevel(loglevel)
|
||
|
|
# Don't add a new handler for every instance initialization (#227), this
|
||
|
|
# would cause duplicated output when the CommandLineInterface as an
|
||
|
|
# normal Python class.
|
||
|
|
if self.log.handlers:
|
||
|
|
handler = self.log.handlers[0]
|
||
|
|
else:
|
||
|
|
handler = logging.StreamHandler()
|
||
|
|
self.log.addHandler(handler)
|
||
|
|
handler.setLevel(loglevel)
|
||
|
|
formatter = logging.Formatter('%(message)s')
|
||
|
|
handler.setFormatter(formatter)
|
||
|
|
|
||
|
|
def _help(self):
|
||
|
|
print(self.parser.format_help())
|
||
|
|
print("commands:")
|
||
|
|
longest = max([len(command) for command in self.commands])
|
||
|
|
format = " %%-%ds %%s" % max(8, longest + 1)
|
||
|
|
commands = sorted(self.commands.items())
|
||
|
|
for name, description in commands:
|
||
|
|
print(format % (name, description))
|
||
|
|
|
||
|
|
def _configure_command(self, cmdname, argv):
|
||
|
|
"""
|
||
|
|
:type cmdname: str
|
||
|
|
:type argv: list[str]
|
||
|
|
"""
|
||
|
|
cmdclass = self.command_classes[cmdname]
|
||
|
|
cmdinst = cmdclass()
|
||
|
|
if self.log:
|
||
|
|
cmdinst.log = self.log # Use our logger, not distutils'.
|
||
|
|
assert isinstance(cmdinst, Command)
|
||
|
|
cmdinst.initialize_options()
|
||
|
|
|
||
|
|
parser = optparse.OptionParser(
|
||
|
|
usage=self.usage % (cmdname, ''),
|
||
|
|
description=self.commands[cmdname]
|
||
|
|
)
|
||
|
|
as_args = getattr(cmdclass, "as_args", ())
|
||
|
|
for long, short, help in cmdclass.user_options:
|
||
|
|
name = long.strip("=")
|
||
|
|
default = getattr(cmdinst, name.replace('-', '_'))
|
||
|
|
strs = ["--%s" % name]
|
||
|
|
if short:
|
||
|
|
strs.append("-%s" % short)
|
||
|
|
strs.extend(cmdclass.option_aliases.get(name, ()))
|
||
|
|
choices = cmdclass.option_choices.get(name, None)
|
||
|
|
if name == as_args:
|
||
|
|
parser.usage += "<%s>" % name
|
||
|
|
elif name in cmdclass.boolean_options:
|
||
|
|
parser.add_option(*strs, action="store_true", help=help)
|
||
|
|
elif name in cmdclass.multiple_value_options:
|
||
|
|
parser.add_option(*strs, action="append", help=help, choices=choices)
|
||
|
|
else:
|
||
|
|
parser.add_option(*strs, help=help, default=default, choices=choices)
|
||
|
|
options, args = parser.parse_args(argv)
|
||
|
|
|
||
|
|
if as_args:
|
||
|
|
setattr(options, as_args.replace('-', '_'), args)
|
||
|
|
|
||
|
|
for key, value in vars(options).items():
|
||
|
|
setattr(cmdinst, key, value)
|
||
|
|
|
||
|
|
try:
|
||
|
|
cmdinst.ensure_finalized()
|
||
|
|
except DistutilsOptionError as err:
|
||
|
|
parser.error(str(err))
|
||
|
|
|
||
|
|
return cmdinst
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
return CommandLineInterface().run(sys.argv)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_mapping(fileobj, filename=None):
|
||
|
|
"""Parse an extraction method mapping from a file-like object.
|
||
|
|
|
||
|
|
>>> buf = StringIO('''
|
||
|
|
... [extractors]
|
||
|
|
... custom = mypackage.module:myfunc
|
||
|
|
...
|
||
|
|
... # Python source files
|
||
|
|
... [python: **.py]
|
||
|
|
...
|
||
|
|
... # Genshi templates
|
||
|
|
... [genshi: **/templates/**.html]
|
||
|
|
... include_attrs =
|
||
|
|
... [genshi: **/templates/**.txt]
|
||
|
|
... template_class = genshi.template:TextTemplate
|
||
|
|
... encoding = latin-1
|
||
|
|
...
|
||
|
|
... # Some custom extractor
|
||
|
|
... [custom: **/custom/*.*]
|
||
|
|
... ''')
|
||
|
|
|
||
|
|
>>> method_map, options_map = parse_mapping(buf)
|
||
|
|
>>> len(method_map)
|
||
|
|
4
|
||
|
|
|
||
|
|
>>> method_map[0]
|
||
|
|
('**.py', 'python')
|
||
|
|
>>> options_map['**.py']
|
||
|
|
{}
|
||
|
|
>>> method_map[1]
|
||
|
|
('**/templates/**.html', 'genshi')
|
||
|
|
>>> options_map['**/templates/**.html']['include_attrs']
|
||
|
|
''
|
||
|
|
>>> method_map[2]
|
||
|
|
('**/templates/**.txt', 'genshi')
|
||
|
|
>>> options_map['**/templates/**.txt']['template_class']
|
||
|
|
'genshi.template:TextTemplate'
|
||
|
|
>>> options_map['**/templates/**.txt']['encoding']
|
||
|
|
'latin-1'
|
||
|
|
|
||
|
|
>>> method_map[3]
|
||
|
|
('**/custom/*.*', 'mypackage.module:myfunc')
|
||
|
|
>>> options_map['**/custom/*.*']
|
||
|
|
{}
|
||
|
|
|
||
|
|
:param fileobj: a readable file-like object containing the configuration
|
||
|
|
text to parse
|
||
|
|
:see: `extract_from_directory`
|
||
|
|
"""
|
||
|
|
extractors = {}
|
||
|
|
method_map = []
|
||
|
|
options_map = {}
|
||
|
|
|
||
|
|
parser = RawConfigParser()
|
||
|
|
parser._sections = OrderedDict(parser._sections) # We need ordered sections
|
||
|
|
|
||
|
|
if PY2:
|
||
|
|
parser.readfp(fileobj, filename)
|
||
|
|
else:
|
||
|
|
parser.read_file(fileobj, filename)
|
||
|
|
|
||
|
|
for section in parser.sections():
|
||
|
|
if section == 'extractors':
|
||
|
|
extractors = dict(parser.items(section))
|
||
|
|
else:
|
||
|
|
method, pattern = [part.strip() for part in section.split(':', 1)]
|
||
|
|
method_map.append((pattern, method))
|
||
|
|
options_map[pattern] = dict(parser.items(section))
|
||
|
|
|
||
|
|
if extractors:
|
||
|
|
for idx, (pattern, method) in enumerate(method_map):
|
||
|
|
if method in extractors:
|
||
|
|
method = extractors[method]
|
||
|
|
method_map[idx] = (pattern, method)
|
||
|
|
|
||
|
|
return method_map, options_map
|
||
|
|
|
||
|
|
|
||
|
|
def parse_keywords(strings=[]):
|
||
|
|
"""Parse keywords specifications from the given list of strings.
|
||
|
|
|
||
|
|
>>> kw = sorted(parse_keywords(['_', 'dgettext:2', 'dngettext:2,3', 'pgettext:1c,2']).items())
|
||
|
|
>>> for keyword, indices in kw:
|
||
|
|
... print((keyword, indices))
|
||
|
|
('_', None)
|
||
|
|
('dgettext', (2,))
|
||
|
|
('dngettext', (2, 3))
|
||
|
|
('pgettext', ((1, 'c'), 2))
|
||
|
|
"""
|
||
|
|
keywords = {}
|
||
|
|
for string in strings:
|
||
|
|
if ':' in string:
|
||
|
|
funcname, indices = string.split(':')
|
||
|
|
else:
|
||
|
|
funcname, indices = string, None
|
||
|
|
if funcname not in keywords:
|
||
|
|
if indices:
|
||
|
|
inds = []
|
||
|
|
for x in indices.split(','):
|
||
|
|
if x[-1] == 'c':
|
||
|
|
inds.append((int(x[:-1]), 'c'))
|
||
|
|
else:
|
||
|
|
inds.append(int(x))
|
||
|
|
indices = tuple(inds)
|
||
|
|
keywords[funcname] = indices
|
||
|
|
return keywords
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
main()
|