Gel4y Mini Shell

View file

File name : localization.py
Content :
# Localization classes and functions
#
# Copyright (C) 2012-2013 Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
# source code or documentation are not subject to the GNU General Public
# License and may only be used or replicated with the express permission of
# Red Hat, Inc.
#

import gettext
import os
import re
import shutil
import langtable
import locale as locale_mod
import glob
from collections import namedtuple

from pyanaconda.core import constants, util
from pyanaconda.core.util import upcase_first_letter, setenv, execWithRedirect
from pyanaconda.modules.common.constants.services import BOSS

from pyanaconda.anaconda_loggers import get_module_logger
log = get_module_logger(__name__)

SCRIPTS_SUPPORTED_BY_CONSOLE = {'Latn', 'Cyrl', 'Grek'}


class LocalizationConfigError(Exception):
    """Exception class for localization configuration related problems"""

    pass


class InvalidLocaleSpec(LocalizationConfigError):
    """Exception class for the errors related to invalid locale specs"""

    pass


def is_valid_langcode(langcode):
    """Check if the given locale has a language specified.

    :return: whether the language or locale is valid
    :rtype: bool
    """
    parsed = langtable.parse_locale(langcode)
    return bool(parsed.language)


def raise_on_invalid_locale(arg):
    """Helper to abort when a locale is not valid.

    :raise: InvalidLocaleSpec
    """
    if not is_valid_langcode(arg):
        raise InvalidLocaleSpec("'{}' is not a valid locale".format(arg))


def get_language_id(locale):
    """Return language id without territory or anything else."""
    return langtable.parse_locale(locale).language


def is_supported_locale(locale):
    """Function that tells if the given locale is supported by the Anaconda or
    not. We consider locales supported by the langtable as supported by the
    Anaconda.

    :param locale: locale to test
    :type locale: str
    :return: whether the given locale is supported or not
    :rtype: bool
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    en_name = get_english_name(locale)
    return bool(en_name)


def locale_supported_in_console(locale):
    """Function that tells if the given locale can be displayed by the Linux console.

    The Linux console can display Latin, Cyrillic and Greek characters reliably,
    but others such as Japanese, can't be correctly installed.

    :param str locale: locale to test
    :return: whether the given locale is supported by the console or not
    :rtype: bool
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    locale_scripts = get_locale_scripts(locale)
    return set(locale_scripts).issubset(SCRIPTS_SUPPORTED_BY_CONSOLE)


def langcode_matches_locale(langcode, locale):
    """Function that tells if the given langcode matches the given locale. I.e. if
    all parts of appearing in the langcode (language, territory, script and
    encoding) are the same as the matching parts of the locale.

    :param langcode: a langcode (e.g. en, en_US, en_US@latin, etc.)
    :type langcode: str
    :param locale: a valid locale (e.g. en_US.UTF-8 or sr_RS.UTF-8@latin, etc.)
    :type locale: str
    :return: whether the given langcode matches the given locale or not
    :rtype: bool
    """
    if not is_valid_langcode(langcode) or not is_valid_langcode(locale):
        # to match, both need to be valid langcodes (need to have at least
        # language specified)
        return False

    langcode_parsed = langtable.parse_locale(langcode)
    locale_parsed = langtable.parse_locale(locale)

    # Check parts one after another. If some part appears in the langcode and
    # doesn't match the one from the locale (or is missing in the locale),
    # return False, otherwise they match
    for lang_val, loc_val in zip(langcode_parsed, locale_parsed):
        if lang_val and lang_val != loc_val:
            return False

    return True


def find_best_locale_match(locale, langcodes):
    """Find the best match for the locale in a list of langcodes. This is useful
    when e.g. pt_BR is a locale and there are possibilities to choose an item
    (e.g. rnote) for a list containing both pt and pt_BR or even also pt_PT.

    :param locale: a valid locale (e.g. en_US.UTF-8 or sr_RS.UTF-8@latin, etc.)
    :type locale: str
    :param langcodes: a list or generator of langcodes (e.g. en, en_US, en_US@latin, etc.)
    :type langcodes: list(str) or generator(str)
    :return: the best matching langcode from the list of None if none matches
    :rtype: str or None
    """
    def score_value_pair(locale_value, langcode_value, weight):
        if locale_value and langcode_value:
            if locale_value == langcode_value:
                # match
                return weight
            else:
                # not match
                return -weight
        elif langcode_value and not locale_value:
            # langcode has something the locale doesn't have
            return -weight
        return 0

    if not is_valid_langcode(locale):
        return None

    scores = []

    # get score for each langcode
    for langcode in langcodes:
        if not is_valid_langcode(langcode):
            scores.append((langcode, 0))
        else:
            locale_parsed = langtable.parse_locale(locale)
            langcode_parsed = langtable.parse_locale(langcode)
            score = score_value_pair(locale_parsed.language, langcode_parsed.language, 1000) + \
                    score_value_pair(locale_parsed.territory, langcode_parsed.territory, 100) + \
                    score_value_pair(locale_parsed.script, langcode_parsed.script, 10) + \
                    score_value_pair(locale_parsed.variant, langcode_parsed.variant, 10) + \
                    score_value_pair(locale_parsed.encoding, langcode_parsed.encoding, 1)
            scores.append((langcode, score))

    # find the best one
    sorted_langcodes = sorted(scores, key=lambda item_score: item_score[1], reverse=True)

    # matches matching only script or encoding or both are not useful
    if sorted_langcodes and sorted_langcodes[0][1] > 100:
        # 100 = requires at least territory to have matched
        return sorted_langcodes[0][0]
    else:
        return None


def setup_locale(locale, localization_proxy=None, text_mode=False):
    """Procedure setting the system to use the given locale and store it in to the
    localization module (if given). DOES NOT PERFORM ANY CHECKS OF THE GIVEN
    LOCALE.

    $LANG must be set by the caller in order to set the language used by gettext.
    Doing this in a thread-safe way is up to the caller.

    We also try to set a proper console font for the locale in text mode.
    If the font for the locale can't be displayed in the Linux console,
    we fall back to the English locale.

    This function returns the locale that was used in the setlocale call, which,
    depending on what the environment and interface is able to support, may be
    different from the locale requested.

    :param str locale: locale to setup
    :param localization_proxy: DBus proxy of the localization module or None
    :param bool text_mode: if the locale is being setup for text mode
    :return: the locale that was actually set
    :rtype: str
    """
    if localization_proxy:
        localization_proxy.SetLanguage(locale)

    # not all locales might be displayable in text mode
    if text_mode:
        # check if the script corresponding to the locale/language
        # can be displayed by the Linux console
        # * all scripts for the given locale/language need to be
        #   supported by the linux console
        # * otherwise users might get a screen full of white rectangles
        #   (also known as "tofu") in text mode
        # then we also need to check if we have information about what
        # font to use for correctly displaying the given language/locale

        script_supported = locale_supported_in_console(locale)
        log.debug("scripts found for locale %s: %s", locale, get_locale_scripts(locale))

        console_fonts = get_locale_console_fonts(locale)
        log.debug("console fonts found for locale %s: %s", locale, console_fonts)

        font_set = False
        if script_supported and console_fonts:
            # try to set console font
            for font in console_fonts:
                if set_console_font(font):
                    # console font set successfully, skip the rest
                    font_set = True
                    break

        if not font_set:
            log.warning("can't set console font for locale %s", locale)
            # report what exactly went wrong
            if not script_supported:
                log.warning("script not supported by console for locale %s", locale)
            if not console_fonts:  # no fonts known for locale
                log.warning("no console font found for locale %s", locale)
            if script_supported and console_fonts:
                log.warning("none of the suggested fonts can be set for locale %s", locale)
            log.warning("falling back to the English locale")
            locale = constants.DEFAULT_LANG
            os.environ["LANG"] = locale  # pylint: disable=environment-modify

    # set the locale to the value we have selected
    # Since glibc does not install all locales, an installable locale may not
    # actually be available right now. Give it a shot and fallback.
    log.debug("setting locale to: %s", locale)
    setenv("LANG", locale)

    try:
        locale_mod.setlocale(locale_mod.LC_ALL, locale)
    except locale_mod.Error as e:
        log.debug("setlocale failed: %s", e)
        locale = constants.DEFAULT_LANG
        setenv("LANG", locale)
        locale_mod.setlocale(locale_mod.LC_ALL, locale)

    set_modules_locale(locale)

    return locale


def set_modules_locale(locale):
    """Set locale of all modules."""
    boss_proxy = BOSS.get_proxy()
    boss_proxy.SetLocale(locale)


def get_english_name(locale):
    """Function returning english name for the given locale.

    :param locale: locale to return english name for
    :type locale: str
    :return: english name for the locale or empty string if unknown
    :rtype: st
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    name = langtable.language_name(languageId=locale, languageIdQuery="en")
    return upcase_first_letter(name)


def get_native_name(locale):
    """Function returning native name for the given locale.

    :param locale: locale to return native name for
    :type locale: str
    :return: english name for the locale or empty string if unknown
    :rtype: st
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    name = langtable.language_name(languageId=locale)
    return upcase_first_letter(name)


def get_available_translations(localedir=None):
    """Method that generates (i.e. returns a generator) available translations
    for the installer in the given localedir.

    :type localedir: str
    :return: generator yielding available translations (languages)
    :rtype: generator yielding strings
    """
    localedir = localedir or gettext._default_localedir

    # usually there are no message files for en
    messagefiles = sorted(glob.glob(localedir + "/*/LC_MESSAGES/anaconda.mo") +
                          ["blob/en/blob/blob"])
    trans_gen = (path.split(os.path.sep)[-3] for path in messagefiles)

    langs = set()

    for trans in trans_gen:
        lang = get_language_id(trans)
        if lang and lang not in langs:
            langs.add(lang)
            # check if there are any locales for the language
            locales = get_language_locales(lang)
            if not locales:
                continue

            yield lang


def get_language_locales(lang):
    """Function returning all locales available for the given language.

    :param lang: language to get available locales for
    :type lang: str
    :return: a list of available locales
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(lang)

    return langtable.list_locales(languageId=lang)


def get_territory_locales(territory):
    """Function returning list of locales for the given territory. The list is
    sorted from the most probable locale to the least probable one (based on
    langtable's ranking.

    :param territory: territory to return locales for
    :type territory: str
    :return: list of locales
    :rtype: list of strings
    """
    return langtable.list_locales(territoryId=territory)


def get_locale_keyboards(locale):
    """Function returning preferred keyboard layouts for the given locale.

    :param locale: locale string
    :type locale: str
    :return: list of preferred keyboard layouts
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_keyboards(languageId=locale)


def get_locale_timezones(locale):
    """Function returning preferred timezones for the given locale.

    :param locale: locale string
    :type locale: str
    :return: list of preferred timezones
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_timezones(languageId=locale)


def get_locale_console_fonts(locale):
    """Function returning preferred console fonts for the given locale.

    :param str locale: locale string
    :return: list of preferred console fonts
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_consolefonts(languageId=locale)


def get_locale_scripts(locale):
    """Function returning preferred scripts (writing systems) for the given locale.

    :param locale: locale string
    :type locale: str
    :return: list of preferred scripts
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_scripts(languageId=locale)


def get_xlated_timezone(tz_spec_part):
    """Function returning translated name of a region, city or complete timezone
    name according to the current value of the $LANG variable.

    :param tz_spec_part: a region, city or complete timezone name
    :type tz_spec_part: str
    :return: translated name of the given region, city or timezone
    :rtype: str
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    locale = os.environ.get("LANG", constants.DEFAULT_LANG)

    raise_on_invalid_locale(locale)

    xlated = langtable.timezone_name(tz_spec_part, languageIdQuery=locale)
    return xlated


def get_firmware_language(text_mode=False):
    """Procedure that returns the firmware language information (if any).

    :param boot text_mode: if the locale is being setup for text mode
    :return: the firmware language translated into a locale string, or None
    :rtype: str
    """
    try:
        n = "/sys/firmware/efi/efivars/PlatformLang-8be4df61-93ca-11d2-aa0d-00e098032b8c"
        with open(n, 'r') as f:
            d = f.read()
    except IOError:
        return None

    # the contents of the file are:
    # 4-bytes of attribute data that we don't care about
    # NUL terminated ASCII string like 'en-US'.
    if len(d) < 10:
        log.debug("PlatformLang was too short")
        return None
    d = d[4:]
    if d[2] != '-':
        log.debug("PlatformLang was malformed")
        return None

    # they use - and we use _, so fix it...
    d = d[:2] + '_' + d[3:-1]

    # UEFI 2.3.1 Errata C specifies 2 aliases in common use that
    # aren't part of RFC 4646, but are allowed in PlatformLang.
    # Because why make anything simple?
    if d.startswith('zh_chs'):
        d = 'zh_Hans'
    elif d.startswith('zh_cht'):
        d = 'zh_Hant'
    d += '.UTF-8'

    if not is_supported_locale(d):
        log.debug("PlatformLang was '%s', which is unsupported.", d)
        return None

    locales = get_language_locales(d)
    if not locales:
        log.debug("No locales found for the PlatformLang '%s'.", d)
        return None

    log.debug("Using UEFI PlatformLang '%s' ('%s') as our language.", d, locales[0])
    return locales[0]


_DateFieldSpec = namedtuple("DateFieldSpec", ["format", "suffix"])


def resolve_date_format(year, month, day, fail_safe=True):
    """Puts the year, month and day objects in the right order according to the
    currently set locale and provides format specification for each of the
    fields.

    :param year: any object or value representing year
    :type year: any
    :param month: any object or value representing month
    :type month: any
    :param day: any object or value representing day
    :type day: any
    :param bool fail_safe: whether to fall back to default in case of invalid
                           format or raise exception instead
    :returns: a pair where the first field contains a tuple with the year, month
              and day objects/values put in the right order and where the second
              field contains a tuple with three :class:`_DateFieldSpec` objects
              specifying formats respectively to the first (year, month, day)
              field, e.g. ((year, month, day), (y_fmt, m_fmt, d_fmt))
    :rtype: tuple
    :raise ValueError: in case currently set locale has unsupported date
                       format and fail_safe is set to False
    """
    FAIL_SAFE_DEFAULT = "%Y-%m-%d"

    def order_terms_formats(fmt_str):
        # see date (1), 'O' (not '0') is a mystery, 'E' is Buddhist calendar, '(.*)'
        # is an arbitrary suffix
        field_spec_re = re.compile(r'([-_0OE^#]*)([yYmbBde])(.*)')

        # see date (1)
        fmt_str = fmt_str.replace("%F", "%Y-%m-%d")

        # e.g. "%d.%m.%Y" -> ['d.', 'm.', 'Y']
        fields = fmt_str.split("%")[1:]

        ordered_terms = []
        ordered_formats = []
        for field in fields:
            match = field_spec_re.match(field)
            if not match:
                # ignore fields we are not interested in (like %A for weekday name, etc.)
                continue

            prefix, item, suffix = match.groups()
            if item in ("d", "e"):
                # "e" is the same as "_d"
                ordered_terms.append(day)
            elif item in ("Y", "y"):
                # 4-digit year, 2-digit year
                ordered_terms.append(year)
            elif item in ("m", "b", "B"):
                # month number, short month name, long month name
                ordered_terms.append(month)

            # "%" + prefix + item gives a format for date/time formatting functions
            ordered_formats.append(_DateFieldSpec("%" + prefix + item, suffix.strip()))

        if len(ordered_terms) != 3 or len(ordered_formats) != 3:
            raise ValueError("Not all fields successfully identified in the format '%s'" % fmt_str)

        return (tuple(ordered_terms), tuple(ordered_formats))

    fmt_str = locale_mod.nl_langinfo(locale_mod.D_FMT)

    if not fmt_str or "%" not in fmt_str:
        if fail_safe:
            # use some sane default
            fmt_str = FAIL_SAFE_DEFAULT
        else:
            raise ValueError("Invalid date format string for current locale: '%s'" % fmt_str)

    try:
        return order_terms_formats(fmt_str)
    except ValueError:
        if not fail_safe:
            raise
        else:
            # if this call fails too, something is going terribly wrong and we
            # should be informed about it
            return order_terms_formats(FAIL_SAFE_DEFAULT)


def set_console_font(font):
    """Try to set console font to the given value.

    :param str font: console font name
    :returns: True on success, False on failure
    :rtype: bool
    """
    log.debug("setting console font to %s", font)
    rc = execWithRedirect("setfont", [font])
    if rc == 0:
        log.debug("console font set successfully to %s", font)
        return True
    else:
        log.error("setting console font to %s failed", font)
        return False


def setup_locale_environment(locale=None, text_mode=False, prefer_environment=False):
    """Clean and configure the local environment variables.

    This function will attempt to determine the desired locale and configure
    the process environment (os.environ) in the least surprising way. If a
    locale argument is provided, it will be attempted first. After that, this
    function will attempt to use the language environment variables in a manner
    similar to gettext(3) (in order, $LANGUAGE, $LC_ALL, $LC_MESSAGES, $LANG),
    followed by the UEFI PlatformLang, followed by a default.

    When this function returns, $LANG will be set, and $LANGUAGE, $LC_ALL,
    and $LC_MESSAGES will not be set, because they get in the way when changing
    the language after startup.

    This function must be run before any threads are started. This function
    modifies the process environment, which is not thread-safe.

    :param str locale: locale to setup if provided
    :param bool text_mode: if the locale is being setup for text mode
    :param bool prefer_environment: whether the process environment, if available, overrides the locale parameter
    :return: None
    :rtype: None
    """
    # pylint: disable=environment-modify

    # Look for a locale in the environment. If the variable is setup but
    # empty it doesn't count, and some programs (KDE) actually do this.
    # If prefer_environment is set, the environment locale can override
    # the parameter passed in. This can be used, for example, by initial-setup,
    # to prefer the possibly-more-recent environment settings before falling back
    # to a locale set at install time and saved in the kickstart.
    if not locale or prefer_environment:
        for varname in ("LANGUAGE", "LC_ALL", "LC_MESSAGES", "LANG"):
            if varname in os.environ and os.environ[varname]:
                locale = os.environ[varname]
                break

    # Look for a locale in the firmware if there was nothing in the environment
    if not locale:
        locale = get_firmware_language(text_mode)

    # parse the locale using langtable
    if locale:
        try:
            env_langs = get_language_locales(locale)
            # the first language is the best match
            locale = env_langs[0]
        except (InvalidLocaleSpec, IndexError):
            log.error("Invalid locale '%s' given on command line, kickstart or environment", locale)
            locale = None

    # If langtable returned no locales, or if nothing was configured, fall back to the default
    if not locale:
        locale = constants.DEFAULT_LANG

    # Save the locale in the environment
    os.environ["LANG"] = locale

    # Cleanup the rest of the environment variables
    for varname in ("LANGUAGE", "LC_ALL", "LC_MESSAGES"):
        if varname in os.environ:
            del os.environ[varname]


def get_locale_map_from_ostree(repo, ref):
    """Get a map of languages and locales from the given OSTree.

    For example: {"en": ["en_US"]}

    :param repo: the OSTree repository url
    :param ref: the name of branch inside the repository
    :return: a map of languages and locales
    """
    # Fallback to just en_US in case of errors.
    locale_map = {"en": ["en_US"]}

    # Let's only handle local embedded repos for now. Anyway, it'd probably
    # not be very common to only override ostreesetup through kickstart and
    # still want the interactive installer. Though to be nice, let's handle
    # that case.
    if not repo.startswith("file://"):
        log.info("ostree repo is not local; defaulting to en_US")
        return

    # Convert to regular UNIX path.
    repo = repo[len("file://"):]

    util.mkdirChain(os.path.join(repo, "tmp/usr/lib"))
    rc = util.execWithRedirect("/usr/bin/ostree",
                               ["checkout", "--repo", repo, ref,
                                "--subpath", "/usr/lib/locale/locale-archive",
                                "%s/tmp/usr/lib/locale" % repo])
    if rc != 0:
        log.error("failed to check out locale-archive; check program.log")
        return

    for line in util.execReadlines("/usr/bin/localedef",
                                   ["--prefix", os.path.join(repo, "tmp"),
                                    "--list-archive"]):
        line = strip_codeset_and_modifier(line)
        if '_' in line:
            (lang, _territory) = line.split('_', 1)
        else:
            lang = line
        if lang not in locale_map:
            locale_map[lang] = [line]
        else:
            locale_map[lang].append(line)

    # Nuke the checkout for good measure.
    shutil.rmtree(os.path.join(repo, "tmp/usr"))
    return locale_map


def strip_codeset_and_modifier(locale):
    """Return a striped version of the given locale.

    :param locale: a name of the locale
    :return: a stripped locale
    """
    if '@' in locale:
        locale = locale[:locale.find('@')]
    if '.' in locale:
        locale = locale[:locale.find('.')]
    return locale