From 4a6290378af8e9ffe16ec24be952b46782b247c3 Mon Sep 17 00:00:00 2001 From: Ondrej Grover Date: Sun, 20 Apr 2014 14:53:01 +0200 Subject: [PATCH] Update the i18n_subsites plugin, addresses many issues Major highlights ................ - fixed and improved cross-linking (fixes #333) with URLs containing e.g. localized month names (thanks to issue getpelican/pelican#1198) - support for custom ``SITEURL`` and ``OUTPUT_PATH`` hierarchy (fixes #182) - sharing of static files (including those of the theme) among subsites (fixes #180) Technical highlights .................... - added a test suite (works with pelican 3.4) - translations are installed into Jinja2 environments of all generators - old locale is restored after generation, fixes autoreload The documentation has been updated and improved (mostly in terms of formatting). Known issues ............ - due to the redesign required for correct cross-linking, older versions of Pelican (<3.4) are not supported, because they lack certain signals - the ``HIDE_UNTRANSLATED_CONTENT`` setting has been deprecated in favor of the ``I18N_UNTRANSLATED_{ARTICLES,PAGES}`` settings which offer more control in order to fix #211. - the test suite works only with pelican 3.4, later versions add a timezone field to the date --- README.rst | 153 +++-- _regenerate_context_helpers.py | 81 --- i18n_subsites.py | 555 +++++++++++++----- implementing_language_buttons.rst | 39 +- localizing_using_jinja2.rst | 116 +++- test_data/content/images/img.png | 0 test_data/content/pages/untranslated-page.rst | 5 + test_data/content/translated_article-cz.rst | 8 + test_data/content/translated_article-de.rst | 8 + test_data/content/translated_article-en.rst | 8 + test_data/content/untranslated_article-en.rst | 9 + test_data/localized_theme/babel.cfg | 2 + test_data/localized_theme/messages.pot | 23 + test_data/localized_theme/static/style.css | 0 test_data/localized_theme/templates/base.html | 7 + .../translations/de/LC_MESSAGES/messages.mo | Bin 0 -> 486 bytes .../translations/de/LC_MESSAGES/messages.po | 23 + test_data/output/an-untranslated-article.html | 50 ++ .../output/cz/an-untranslated-article-en.html | 49 ++ test_data/output/cz/feeds_all.atom.xml | 10 + test_data/output/cz/index.html | 54 ++ test_data/output/cz/translated-article.html | 52 ++ .../de/drafts/an-untranslated-article-en.html | 49 ++ test_data/output/de/feeds_all.atom.xml | 8 + test_data/output/de/index.html | 42 ++ .../output/de/pages/untranslated-page-en.html | 30 + test_data/output/de/translated-article.html | 52 ++ test_data/output/feeds_all.atom.xml | 10 + test_data/output/images/img.png | 0 test_data/output/index.html | 55 ++ test_data/output/pages/untranslated-page.html | 31 + test_data/output/theme/style.css | 0 test_data/output/translated-article.html | 53 ++ test_data/pelicanconf.py | 53 ++ test_i18n_subsites.py | 139 +++++ 35 files changed, 1465 insertions(+), 309 deletions(-) delete mode 100644 _regenerate_context_helpers.py create mode 100644 test_data/content/images/img.png create mode 100644 test_data/content/pages/untranslated-page.rst create mode 100644 test_data/content/translated_article-cz.rst create mode 100644 test_data/content/translated_article-de.rst create mode 100644 test_data/content/translated_article-en.rst create mode 100644 test_data/content/untranslated_article-en.rst create mode 100644 test_data/localized_theme/babel.cfg create mode 100644 test_data/localized_theme/messages.pot create mode 100644 test_data/localized_theme/static/style.css create mode 100644 test_data/localized_theme/templates/base.html create mode 100644 test_data/localized_theme/translations/de/LC_MESSAGES/messages.mo create mode 100644 test_data/localized_theme/translations/de/LC_MESSAGES/messages.po create mode 100644 test_data/output/an-untranslated-article.html create mode 100644 test_data/output/cz/an-untranslated-article-en.html create mode 100644 test_data/output/cz/feeds_all.atom.xml create mode 100644 test_data/output/cz/index.html create mode 100644 test_data/output/cz/translated-article.html create mode 100644 test_data/output/de/drafts/an-untranslated-article-en.html create mode 100644 test_data/output/de/feeds_all.atom.xml create mode 100644 test_data/output/de/index.html create mode 100644 test_data/output/de/pages/untranslated-page-en.html create mode 100644 test_data/output/de/translated-article.html create mode 100644 test_data/output/feeds_all.atom.xml create mode 100644 test_data/output/images/img.png create mode 100644 test_data/output/index.html create mode 100644 test_data/output/pages/untranslated-page.html create mode 100644 test_data/output/theme/style.css create mode 100644 test_data/output/translated-article.html create mode 100644 test_data/pelicanconf.py create mode 100644 test_i18n_subsites.py diff --git a/README.rst b/README.rst index 37ceef0..12e113d 100644 --- a/README.rst +++ b/README.rst @@ -1,73 +1,156 @@ -====================== +======================= I18N Sub-sites Plugin -====================== +======================= -This plugin extends the translations functionality by creating internationalized sub-sites for the default site. It is therefore redundant with the *\*_LANG_{SAVE_AS,URL}* variables, so it disables them to prevent conflicts. +This plugin extends the translations functionality by creating +internationalized sub-sites for the default site. + +This plugin is designed for Pelican 3.4 and later. What it does ============ -1. The *\*_LANG_URL* and *\*_LANG_SAVE_AS* variables are set to their normal counterparts (e.g. *ARTICLE_URL*) so they don't conflict with this scheme. -2. While building the site for *DEFAULT_LANG* the translations of pages and articles are not generated, but their relations to the original content is kept via links to them. -3. For each non-default language a "sub-site" with a modified config [#conf]_ is created [#run]_, linking the translations to the originals (if available). The configured language code is appended to the *OUTPUT_PATH* and *SITEURL* of each sub-site. For each sub-site, *DEFAULT_LANG* is changed to the language of the sub-site so that articles in a different language are treated as translations. -If *HIDE_UNTRANSLATED_CONTENT* is True (default), content without a translation for a language is generated as hidden (for pages) or draft (for articles) for the corresponding language sub-site. +1. When the content of the main site is being generated, the settings + are saved and the generation stops when content is ready to be + written. While reading source files and generating content objects, + the output queue is modified in certain ways: -.. [#conf] For each language a config override is given in the *I18N_SUBSITES* dictionary. -.. [#run] Using a new *PELICAN_CLASS* instance and its ``run`` method, so each sub-site could even have a different *PELICAN_CLASS* if specified in *I18N_SUBSITES* conf overrides. + - translations that will appear as native in a different (sub-)site + will be removed + - untranslated articles will be transformed to drafts if + ``I18N_UNTRANSLATED_ARTICLES`` is ``'hide'`` (default), removed if + ``'remove'`` or kept as they are if ``'keep'``. + - untranslated pages will be transformed into hidden pages if + ``I18N_UNTRANSLATED_PAGES`` is ``'hide'`` (default), removed if + ``'remove'`` or kept as they are if ``'keep'``.'' + - additional content manipulation similar to articles and pages can + be specified for custom generators in the ``I18N_GENERATOR_INFO`` + setting. + +2. For each language specified in the ``I18N_SUBSITES`` dictionary the + settings overrides are applied to the settings from the main site + and a new sub-site is generated in the same way as with the main + site until content is ready to be written. +3. When all (sub-)sites are waiting for content writing, all removed + contents, translations and static files are interlinked across the + (sub-)sites. +4. Finally, all the output is written. Setting it up ============= -For each extra used language code, a language-specific variables overrides dictionary must be given (but can be empty) in the *I18N_SUBSITES* dictionary +For each extra used language code, a language-specific settings overrides +dictionary must be given (but can be empty) in the ``I18N_SUBSITES`` dictionary .. code-block:: python PLUGINS = ['i18n_subsites', ...] - # mapping: language_code -> conf_overrides_dict + # mapping: language_code -> settings_overrides_dict I18N_SUBSITES = { 'cz': { 'SITENAME': 'Hezkej blog', } } -- The language code is the language identifier used in the *lang* metadata. It is appended to *OUTPUT_PATH* and *SITEURL* of each I18N sub-site. -- The internationalized config overrides dictionary may specify configuration variable overrides — e.g. a different *LOCALE*, *SITENAME*, *TIMEZONE*, etc. However, it **must not** override *OUTPUT_PATH* and *SITEURL* as they are modified automatically by appending the language code. +Default and special overrides +----------------------------- +The settings overrides may contain arbitrary settings, however, there +are some that are handled in a special way: + +``SITEURL`` + Any overrides to this setting should ensure that there is some level + of hierarchy between all (sub-)sites, because Pelican makes all URLs + relative to ``SITEURL`` and the plugin can only cross-link between + the sites using this hierarchy. For instance, with the main site + ``http://example.com`` a sub-site ``http://example.com/de`` will + work, but ``http://de.example.com`` will not. If not overridden, the + language code (the language identifier used in the ``lang`` + metadata) is appended to the main ``SITEURL`` for each sub-site. +``OUTPUT_PATH``, ``CACHE_PATH`` + If not overridden, the language code is appended as with ``SITEURL``. + Separate cache paths are required as parser results depend on the locale. +``STATIC_PATHS``, ``THEME_STATIC_PATHS`` + If not overridden, they are set to ``[]`` and all links to static + files are cross-linked to the main site. +``THEME``, ``THEME_STATIC_DIR`` + If overridden, the logic with ``THEME_STATIC_PATHS`` does not apply. +``DEFAULT_LANG`` + This should not be overridden as the plugin changes it to the + language code of each sub-site to change what is perceived as translations. Localizing templates -------------------- -Most importantly, this plugin can use localized templates for each sub-site. There are two approaches to having the templates localized: +Most importantly, this plugin can use localized templates for each +sub-site. There are two approaches to having the templates localized: -- You can set a different *THEME* override for each language in *I18N_SUBSITES*, e.g. by making a copy of a theme ``my_theme`` to ``my_theme_lang`` and then editing the templates in the new localized theme. This approach means you don't have to deal with gettext ``*.po`` files, but it is harder to maintain over time. -- You use only one theme and localize the templates using the `jinja2.ext.i18n Jinja2 extension `_. For a kickstart read this `guide <./localizing_using_jinja2.rst>`_. +- You can set a different ``THEME`` override for each language in + ``I18N_SUBSITES``, e.g. by making a copy of a theme ``my_theme`` to + ``my_theme_lang`` and then editing the templates in the new + localized theme. This approach means you don't have to deal with + gettext ``*.po`` files, but it is harder to maintain over time. +- You use only one theme and localize the templates using the + `jinja2.ext.i18n Jinja2 extension + `_. For a kickstart + read this `guide <./localizing_using_jinja2.rst>`_. -It may be convenient to add language buttons to your theme in addition to the translation links of articles and pages. These buttons could, for example, point to the *SITEURL* of each (sub-)site. For this reason the plugin adds these variables to the template context: +Additional context variables +............................ -main_lang - The language of the top-level site — the original *DEFAULT_LANG* -main_siteurl - The *SITEURL* of the top-level site — the original *SITEURL* -lang_siteurls - An ordered dictionary, mapping all used languages to their *SITEURL*. The ``main_lang`` is the first key with ``main_siteurl`` as the value. This dictionary is useful for implementing global language buttons that show the language of the currently viewed (sub-)site too. -extra_siteurls - An ordered dictionary, subset of ``lang_siteurls``, the current *DEFAULT_LANG* of the rendered (sub-)site is not included, so for each (sub-)site ``set(extra_siteurls) == set(lang_siteurls) - set([DEFAULT_LANG])``. This dictionary is useful for implementing global language buttons that do not show the current language. +It may be convenient to add language buttons to your theme in addition +to the translation links of articles and pages. These buttons could, +for example, point to the ``SITEURL`` of each (sub-)site. For this +reason the plugin adds these variables to the template context: -If you don't like the default ordering of the ordered dictionaries, use a Jinja2 filter to alter the ordering. +``main_lang`` + The language of the main site — the original ``DEFAULT_LANG`` +``main_siteurl`` + The ``SITEURL`` of the main site — the original ``SITEURL`` +``lang_siteurls`` + An ordered dictionary, mapping all used languages to their + ``SITEURL``. The ``main_lang`` is the first key with ``main_siteurl`` + as the value. This dictionary is useful for implementing global + language buttons that show the language of the currently viewed + (sub-)site too. +``extra_siteurls`` + An ordered dictionary, subset of ``lang_siteurls``, the current + ``DEFAULT_LANG`` of the rendered (sub-)site is not included, so for + each (sub-)site ``set(extra_siteurls) == set(lang_siteurls) - + set([DEFAULT_LANG])``. This dictionary is useful for implementing + global language buttons that do not show the current language. +``relpath_to_site`` + A function that returns a relative path from the first (sub-)site to + the second (sub-)site where the (sub-)sites are identified by the + language codes given as two arguments. -This short `howto <./implementing_language_buttons.rst>`_ shows two example implementations of language buttons. +If you don't like the default ordering of the ordered dictionaries, +use a Jinja2 filter to alter the ordering. + +All the siteurls above are always absolute even in the case of +``RELATIVE_URLS == True`` (it would be to complicated to replicate the +Pelican internals for local siteurls), so you may rather use something +like ``{{ SITEURL }}/{{ relpath_to_site(DEFAULT_LANG, main_lang }}`` +to link to the main site. + +This short `howto <./implementing_language_buttons.rst>`_ shows two +example implementations of language buttons. Usage notes =========== -- It is **mandatory** to specify *lang* metadata for each article and page as *DEFAULT_LANG* is later changed for each sub-site, so content without *lang* metadata woudl be rendered in every (sub-)site. -- As with the original translations functionality, *slug* metadata is used to group translations. It is therefore often convenient to compensate for this by overriding the content URL (which defaults to slug) using the *URL* and *Save_as* metadata. - -Future plans -============ - -- add a test suite +- It is **mandatory** to specify ``lang`` metadata for each article + and page as ``DEFAULT_LANG`` is later changed for each sub-site, so + content without ``lang`` metadata would be rendered in every + (sub-)site. +- As with the original translations functionality, ``slug`` metadata + is used to group translations. It is therefore often convenient to + compensate for this by overriding the content URL (which defaults to + slug) using the ``url`` and ``save_as`` metadata. You could also + give articles e.g. ``name`` metadata and use it in ``ARTICLE_URL = + '{name}.html'``. Development =========== -- A demo and test site is in the ``gh-pages`` branch and can be seen at http://smartass101.github.io/pelican-plugins/ +- A demo and a test site is in the ``gh-pages`` branch and can be seen + at http://smartass101.github.io/pelican-plugins/ diff --git a/_regenerate_context_helpers.py b/_regenerate_context_helpers.py deleted file mode 100644 index 556f448..0000000 --- a/_regenerate_context_helpers.py +++ /dev/null @@ -1,81 +0,0 @@ - -import math -import random -from collections import defaultdict -from operator import attrgetter, itemgetter - - -def regenerate_context_articles(generator): - """Helper to regenerate context after modifying articles draft state - - essentially just a copy from pelican.generators.ArticlesGenerator.generate_context - after process_translations up to signal sending - - This has to be kept in sync untill a better solution is found - This is for Pelican version 3.3.0 - """ - # Simulate __init__ for fields that need it - generator.dates = {} - generator.tags = defaultdict(list) - generator.categories = defaultdict(list) - generator.authors = defaultdict(list) - - - # Simulate ArticlesGenerator.generate_context - for article in generator.articles: - # only main articles are listed in categories and tags - # not translations - generator.categories[article.category].append(article) - if hasattr(article, 'tags'): - for tag in article.tags: - generator.tags[tag].append(article) - # ignore blank authors as well as undefined - if hasattr(article, 'author') and article.author.name != '': - generator.authors[article.author].append(article) - - - # sort the articles by date - generator.articles.sort(key=attrgetter('date'), reverse=True) - generator.dates = list(generator.articles) - generator.dates.sort(key=attrgetter('date'), - reverse=generator.context['NEWEST_FIRST_ARCHIVES']) - - # create tag cloud - tag_cloud = defaultdict(int) - for article in generator.articles: - for tag in getattr(article, 'tags', []): - tag_cloud[tag] += 1 - - tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) - tag_cloud = tag_cloud[:generator.settings.get('TAG_CLOUD_MAX_ITEMS')] - - tags = list(map(itemgetter(1), tag_cloud)) - if tags: - max_count = max(tags) - steps = generator.settings.get('TAG_CLOUD_STEPS') - - # calculate word sizes - generator.tag_cloud = [ - ( - tag, - int(math.floor(steps - (steps - 1) * math.log(count) - / (math.log(max_count)or 1))) - ) - for tag, count in tag_cloud - ] - # put words in chaos - random.shuffle(generator.tag_cloud) - - # and generate the output :) - - # order the categories per name - generator.categories = list(generator.categories.items()) - generator.categories.sort( - reverse=generator.settings['REVERSE_CATEGORY_ORDER']) - - generator.authors = list(generator.authors.items()) - generator.authors.sort() - - generator._update_context(('articles', 'dates', 'tags', 'categories', - 'tag_cloud', 'authors', 'related_posts')) - diff --git a/i18n_subsites.py b/i18n_subsites.py index 2ae30e7..ae2066c 100644 --- a/i18n_subsites.py +++ b/i18n_subsites.py @@ -1,189 +1,440 @@ -"""i18n_subsites plugin creates i18n-ized subsites of the default site""" +"""i18n_subsites plugin creates i18n-ized subsites of the default site +This plugin is designed for Pelican 3.4 and later +""" import os import six import logging +import posixpath + +from copy import copy from itertools import chain -from collections import defaultdict, OrderedDict +from operator import attrgetter +from collections import OrderedDict +from contextlib import contextmanager +from six.moves.urllib.parse import urlparse import gettext +import locale from pelican import signals -from pelican.contents import Page, Article +from pelican.generators import ArticlesGenerator, PagesGenerator from pelican.settings import configure_settings - -from ._regenerate_context_helpers import regenerate_context_articles - +from pelican.contents import Draft # Global vars -_main_site_generated = False -_main_site_lang = "en" -_main_siteurl = '' -_lang_siteurls = None -logger = logging.getLogger(__name__) +_MAIN_SETTINGS = None # settings dict of the main Pelican instance +_MAIN_LANG = None # lang of the main Pelican instance +_MAIN_SITEURL = None # siteurl of the main Pelican instance +_MAIN_STATIC_FILES = None # list of Static instances the main Pelican instance +_SUBSITE_QUEUE = {} # map: lang -> settings overrides +_SITE_DB = OrderedDict() # OrderedDict: lang -> siteurl +_SITES_RELPATH_DB = {} # map: (lang, base_lang) -> relpath +# map: generator -> list of removed contents that need interlinking +_GENERATOR_DB = {} +_NATIVE_CONTENT_URL_DB = {} # map: source_path -> content in its native lang +_LOGGER = logging.getLogger(__name__) +@contextmanager +def temporary_locale(temp_locale=None): + '''Enable code to run in a context with a temporary locale -def disable_lang_vars(pelican_obj): - """Set lang specific url and save_as vars to the non-lang defaults - - e.g. ARTICLE_LANG_URL = ARTICLE_URL - They would conflict with this plugin otherwise - """ - global _main_site_lang, _main_siteurl, _lang_siteurls - s = pelican_obj.settings - for content in ['ARTICLE', 'PAGE']: - for meta in ['_URL', '_SAVE_AS']: - s[content + '_LANG' + meta] = s[content + meta] - if not _main_site_generated: - _main_site_lang = s['DEFAULT_LANG'] - _main_siteurl = s['SITEURL'] - _lang_siteurls = [(lang, _main_siteurl + '/' + lang) for lang in s.get('I18N_SUBSITES', {}).keys()] - # To be able to use url for main site root when SITEURL == '' (e.g. when developing) - _lang_siteurls = [(_main_site_lang, ('/' if _main_siteurl == '' else _main_siteurl))] + _lang_siteurls - _lang_siteurls = OrderedDict(_lang_siteurls) - - - -def create_lang_subsites(pelican_obj): - """For each language create a subsite using the lang-specific config - - for each generated lang append language subpath to SITEURL and OUTPUT_PATH - and set DEFAULT_LANG to the language code to change perception of what is translated - and set DELETE_OUTPUT_DIRECTORY to False to prevent deleting output from previous runs - Then generate the subsite using a PELICAN_CLASS instance and its run method. - """ - global _main_site_generated - if _main_site_generated: # make sure this is only called once - return - else: - _main_site_generated = True - - orig_settings = pelican_obj.settings - for lang, overrides in orig_settings.get('I18N_SUBSITES', {}).items(): - settings = orig_settings.copy() - settings.update(overrides) - settings['SITEURL'] = _lang_siteurls[lang] - settings['OUTPUT_PATH'] = os.path.join(orig_settings['OUTPUT_PATH'], lang, '') - settings['DEFAULT_LANG'] = lang # to change what is perceived as translations - settings['DELETE_OUTPUT_DIRECTORY'] = False # prevent deletion of previous runs - settings = configure_settings(settings) # to set LOCALE, etc. - - cls = settings['PELICAN_CLASS'] - if isinstance(cls, six.string_types): - module, cls_name = cls.rsplit('.', 1) - module = __import__(module) - cls = getattr(module, cls_name) - - pelican_obj = cls(settings) - logger.debug("Generating i18n subsite for lang '{}' using class '{}'".format(lang, str(cls))) - pelican_obj.run() - _main_site_generated = False # for autoreload mode + Resets the locale back when exiting context. + Can set a temporary locale if provided + ''' + orig_locale = locale.setlocale(locale.LC_ALL) + if temp_locale is not None: + locale.setlocale(locale.LC_ALL, temp_locale) + yield + locale.setlocale(locale.LC_ALL, orig_locale) +def initialize_dbs(settings): + '''Initialize internal DBs using the Pelican settings dict -def move_translations_links(content_object): - """This function points translations links to the sub-sites + This clears the DBs for e.g. autoreload mode to work + ''' + global _MAIN_SETTINGS, _MAIN_SITEURL, _MAIN_LANG, _SUBSITE_QUEUE + _MAIN_SETTINGS = settings + _MAIN_LANG = settings['DEFAULT_LANG'] + _MAIN_SITEURL = settings['SITEURL'] + _SUBSITE_QUEUE = settings.get('I18N_SUBSITES', {}).copy() + prepare_site_db_and_overrides() + # clear databases in case of autoreload mode + _SITES_RELPATH_DB.clear() + _NATIVE_CONTENT_URL_DB.clear() + _GENERATOR_DB.clear() - by prepending their location with the language code - or directs an original DEFAULT_LANG translation back to top level site - """ - for translation in content_object.translations: - if translation.lang == _main_site_lang: - # cannot prepend, must take to top level - lang_prepend = '../' + +def prepare_site_db_and_overrides(): + '''Prepare overrides and create _SITE_DB + + _SITE_DB.keys() need to be ready for filter_translations + ''' + _SITE_DB.clear() + _SITE_DB[_MAIN_LANG] = _MAIN_SITEURL + # make sure it works for both root-relative and absolute + main_siteurl = '/' if _MAIN_SITEURL == '' else _MAIN_SITEURL + for lang, overrides in _SUBSITE_QUEUE.items(): + if 'SITEURL' not in overrides: + overrides['SITEURL'] = posixpath.join(main_siteurl, lang) + _SITE_DB[lang] = overrides['SITEURL'] + # default subsite hierarchy + if 'OUTPUT_PATH' not in overrides: + overrides['OUTPUT_PATH'] = os.path.join( + _MAIN_SETTINGS['OUTPUT_PATH'], lang) + if 'CACHE_PATH' not in overrides: + overrides['CACHE_PATH'] = os.path.join( + _MAIN_SETTINGS['CACHE_PATH'], lang) + if 'STATIC_PATHS' not in overrides: + overrides['STATIC_PATHS'] = [] + if ('THEME' not in overrides and 'THEME_STATIC_DIR' not in overrides and + 'THEME_STATIC_PATHS' not in overrides): + relpath = relpath_to_site(lang, _MAIN_LANG) + overrides['THEME_STATIC_DIR'] = posixpath.join( + relpath, _MAIN_SETTINGS['THEME_STATIC_DIR']) + overrides['THEME_STATIC_PATHS'] = [] + # to change what is perceived as translations + overrides['DEFAULT_LANG'] = lang + + +def subscribe_filter_to_signals(settings): + '''Subscribe content filter to requested signals''' + for sig in settings.get('I18N_FILTER_SIGNALS', []): + sig.connect(filter_contents_translations) + + +def initialize_plugin(pelican_obj): + '''Initialize plugin variables and Pelican settings''' + if _MAIN_SETTINGS is None: + initialize_dbs(pelican_obj.settings) + subscribe_filter_to_signals(pelican_obj.settings) + + +def get_site_path(url): + '''Get the path component of an url, excludes siteurl + + also normalizes '' to '/' for relpath to work, + otherwise it could be interpreted as a relative filesystem path + ''' + path = urlparse(url).path + if path == '': + path = '/' + return path + + +def relpath_to_site(lang, target_lang): + '''Get relative path from siteurl of lang to siteurl of base_lang + + the output is cached in _SITES_RELPATH_DB + ''' + path = _SITES_RELPATH_DB.get((lang, target_lang), None) + if path is None: + siteurl = _SITE_DB.get(lang, _MAIN_SITEURL) + target_siteurl = _SITE_DB.get(target_lang, _MAIN_SITEURL) + path = posixpath.relpath(get_site_path(target_siteurl), + get_site_path(siteurl)) + _SITES_RELPATH_DB[(lang, target_lang)] = path + return path + + +def save_generator(generator): + '''Save the generator for later use + + initialize the removed content list + ''' + _GENERATOR_DB[generator] = [] + + +def article2draft(article): + '''Transform an Article to Draft''' + draft = Draft(article._content, article.metadata, article.settings, + article.source_path, article._context) + draft.status = 'draft' + return draft + + +def page2hidden_page(page): + '''Transform a Page to a hidden Page''' + page.status = 'hidden' + return page + + +class GeneratorInspector(object): + '''Inspector of generator instances''' + + generators_info = { + ArticlesGenerator: { + 'translations_lists': ['translations', 'drafts_translations'], + 'contents_lists': [('articles', 'drafts')], + 'hiding_func': article2draft, + 'policy': 'I18N_UNTRANSLATED_ARTICLES', + }, + PagesGenerator: { + 'translations_lists': ['translations', 'hidden_translations'], + 'contents_lists': [('pages', 'hidden_pages')], + 'hiding_func': page2hidden_page, + 'policy': 'I18N_UNTRANSLATED_PAGES', + }, + } + + def __init__(self, generator): + '''Identify the best known class of the generator instance + + The class ''' + self.generator = generator + self.generators_info.update(generator.settings.get( + 'I18N_GENERATORS_INFO', {})) + for cls in generator.__class__.__mro__: + if cls in self.generators_info: + self.info = self.generators_info[cls] + break else: - lang_prepend = translation.lang + '/' - translation.override_url = lang_prepend + translation.url + self.info = {} + + def translations_lists(self): + '''Iterator over lists of content translations''' + return (getattr(self.generator, name) for name in + self.info.get('translations_lists', [])) + + def contents_list_pairs(self): + '''Iterator over pairs of normal and hidden contents''' + return (tuple(getattr(self.generator, name) for name in names) + for names in self.info.get('contents_lists', [])) + + def hiding_function(self): + '''Function for transforming content to a hidden version''' + hiding_func = self.info.get('hiding_func', lambda x: x) + return hiding_func + + def untranslated_policy(self, default): + '''Get the policy for untranslated content''' + return self.generator.settings.get(self.info.get('policy', None), + default) + + def all_contents(self): + '''Iterator over all contents''' + translations_iterator = chain(*self.translations_lists()) + return chain(translations_iterator, + *(pair[i] for pair in self.contents_list_pairs() + for i in (0, 1))) +def filter_contents_translations(generator): + '''Filter the content and translations lists of a generator -def update_generator_contents(generator, *args): - """Update the contents lists of a generator + Filters out + 1) translations which will be generated in a different site + 2) content that is not in the language of the currently + generated site but in that of a different site, content in a + language which has no site is generated always. The filtering + method bay be modified by the respective untranslated policy + ''' + inspector = GeneratorInspector(generator) + current_lang = generator.settings['DEFAULT_LANG'] + langs_with_sites = _SITE_DB.keys() + removed_contents = _GENERATOR_DB[generator] - Empty the (hidden_)translation attribute of article and pages generators - to prevent generating the translations as they will be generated in the lang sub-site - and point the content translations links to the sub-sites - - Hide content without a translation for current DEFAULT_LANG - if HIDE_UNTRANSLATED_CONTENT is True - """ - generator.translations = [] - is_pages_gen = hasattr(generator, 'pages') - if is_pages_gen: - generator.hidden_translations = [] - for page in chain(generator.pages, generator.hidden_pages): - move_translations_links(page) - else: # is an article generator - for article in chain(generator.articles, generator.drafts): - move_translations_links(article) - - if not generator.settings.get('HIDE_UNTRANSLATED_CONTENT', True): - return - contents = generator.pages if is_pages_gen else generator.articles - hidden_contents = generator.hidden_pages if is_pages_gen else generator.drafts - default_lang = generator.settings['DEFAULT_LANG'] - for content_object in contents[:]: # loop over copy for removing - if content_object.lang != default_lang: - if isinstance(content_object, Article): - content_object.status = 'draft' - elif isinstance(content_object, Page): - content_object.status = 'hidden' - contents.remove(content_object) - hidden_contents.append(content_object) - if not is_pages_gen: # regenerate categories, tags, etc. for articles - if hasattr(generator, '_generate_context_aggregate'): # if implemented - # Simulate __init__ for fields that need it - generator.dates = {} - generator.tags = defaultdict(list) - generator.categories = defaultdict(list) - generator.authors = defaultdict(list) - generator._generate_context_aggregate() - else: # fallback for Pelican 3.3.0 - regenerate_context_articles(generator) + for translations in inspector.translations_lists(): + for translation in translations[:]: # copy to be able to remove + if translation.lang in langs_with_sites: + translations.remove(translation) + removed_contents.append(translation) + hiding_func = inspector.hiding_function() + untrans_policy = inspector.untranslated_policy(default='hide') + for (contents, other_contents) in inspector.contents_list_pairs(): + for content in contents[:]: # copy for removing in loop + if content.lang == current_lang: # in native lang + # save the native URL attr formatted in the current locale + _NATIVE_CONTENT_URL_DB[content.source_path] = content.url + elif content.lang in langs_with_sites and untrans_policy != 'keep': + contents.remove(content) + if untrans_policy == 'hide': + other_contents.append(hiding_func(content)) + elif untrans_policy == 'remove': + removed_contents.append(content) def install_templates_translations(generator): - """Install gettext translations for current DEFAULT_LANG in the jinja2.Environment + '''Install gettext translations in the jinja2.Environment - if the 'jinja2.ext.i18n' jinja2 extension is enabled - adds some useful variables into the template context - """ - generator.context['main_siteurl'] = _main_siteurl - generator.context['main_lang'] = _main_site_lang - generator.context['lang_siteurls'] = _lang_siteurls - current_def_lang = generator.settings['DEFAULT_LANG'] - extra_siteurls = _lang_siteurls.copy() - extra_siteurls.pop(current_def_lang) - generator.context['extra_siteurls'] = extra_siteurls - - if 'jinja2.ext.i18n' not in generator.settings['JINJA_EXTENSIONS']: - return - domain = generator.settings.get('I18N_GETTEXT_DOMAIN', 'messages') - localedir = generator.settings.get('I18N_GETTEXT_LOCALEDIR') - if localedir is None: - localedir = os.path.join(generator.theme, 'translations') - if current_def_lang == generator.settings.get('I18N_TEMPLATES_LANG', _main_site_lang): - translations = gettext.NullTranslations() - else: - languages = [current_def_lang] - try: - translations = gettext.translation(domain, localedir, languages) - except (IOError, OSError): - logger.error("Cannot find translations for language '{}' in '{}' with domain '{}'. Installing NullTranslations.".format(languages[0], localedir, domain)) + Only if the 'jinja2.ext.i18n' jinja2 extension is enabled + the translations for the current DEFAULT_LANG are installed. + ''' + if 'jinja2.ext.i18n' in generator.settings['JINJA_EXTENSIONS']: + domain = generator.settings.get('I18N_GETTEXT_DOMAIN', 'messages') + localedir = generator.settings.get('I18N_GETTEXT_LOCALEDIR') + if localedir is None: + localedir = os.path.join(generator.theme, 'translations') + current_lang = generator.settings['DEFAULT_LANG'] + if current_lang == generator.settings.get('I18N_TEMPLATES_LANG', + _MAIN_LANG): translations = gettext.NullTranslations() - newstyle = generator.settings.get('I18N_GETTEXT_NEWSTYLE', True) - generator.env.install_gettext_translations(translations, newstyle) + else: + langs = [current_lang] + try: + translations = gettext.translation(domain, localedir, langs) + except (IOError, OSError): + _LOGGER.error(( + "Cannot find translations for language '{}' in '{}' with " + "domain '{}'. Installing NullTranslations.").format( + langs[0], localedir, domain)) + translations = gettext.NullTranslations() + newstyle = generator.settings.get('I18N_GETTEXT_NEWSTYLE', True) + generator.env.install_gettext_translations(translations, newstyle) +def add_variables_to_context(generator): + '''Adds useful iterable variables to template context''' + context = generator.context # minimize attr lookup + context['relpath_to_site'] = relpath_to_site + context['main_siteurl'] = _MAIN_SITEURL + context['main_lang'] = _MAIN_LANG + context['lang_siteurls'] = _SITE_DB + current_lang = generator.settings['DEFAULT_LANG'] + extra_siteurls = _SITE_DB.copy() + extra_siteurls.pop(current_lang) + context['extra_siteurls'] = extra_siteurls + + +def interlink_translations(content): + '''Link content to translations in their main language + + so the URL (including localized month names) of the different subsites + will be honored + ''' + lang = content.lang + # sort translations by lang + content.translations.sort(key=attrgetter('lang')) + for translation in content.translations: + relpath = relpath_to_site(lang, translation.lang) + url = _NATIVE_CONTENT_URL_DB[translation.source_path] + translation.override_url = posixpath.join(relpath, url) + + +def interlink_translated_content(generator): + '''Make translations link to the native locations + + for generators that may contain translated content + ''' + inspector = GeneratorInspector(generator) + for content in inspector.all_contents(): + interlink_translations(content) + + +def interlink_removed_content(generator): + '''For all contents removed from generation queue update interlinks + + link to the native location + ''' + current_lang = generator.settings['DEFAULT_LANG'] + for content in _GENERATOR_DB[generator]: + url = _NATIVE_CONTENT_URL_DB[content.source_path] + relpath = relpath_to_site(current_lang, content.lang) + content.override_url = posixpath.join(relpath, url) + + +def interlink_static_files(generator): + '''Add links to static files in the main site if necessary''' + if generator.settings['STATIC_PATHS'] != []: + return # customized STATIC_PATHS + filenames = generator.context['filenames'] # minimize attr lookup + relpath = relpath_to_site(generator.settings['DEFAULT_LANG'], _MAIN_LANG) + for staticfile in _MAIN_STATIC_FILES: + if staticfile.get_relative_source_path() not in filenames: + staticfile = copy(staticfile) # prevent override in main site + staticfile.override_url = posixpath.join(relpath, staticfile.url) + generator.add_source_path(staticfile) + + +def save_main_static_files(static_generator): + '''Save the static files generated for the main site''' + global _MAIN_STATIC_FILES + # test just for current lang as settings change in autoreload mode + if static_generator.settings['DEFAULT_LANG'] == _MAIN_LANG: + _MAIN_STATIC_FILES = static_generator.staticfiles + + +def update_generators(): + '''Update the context of all generators + + Ads useful variables and translations into the template context + and interlink translations + ''' + for generator in _GENERATOR_DB.keys(): + install_templates_translations(generator) + add_variables_to_context(generator) + interlink_static_files(generator) + interlink_removed_content(generator) + interlink_translated_content(generator) + + +def get_pelican_cls(settings): + '''Get the Pelican class requested in settings''' + cls = settings['PELICAN_CLASS'] + if isinstance(cls, six.string_types): + module, cls_name = cls.rsplit('.', 1) + module = __import__(module) + cls = getattr(module, cls_name) + return cls + + +def create_next_subsite(pelican_obj): + '''Create the next subsite using the lang-specific config + + If there are no more subsites in the generation queue, update all + the generators (interlink translations and removed content, add + variables and translations to template context). Otherwise get the + language and overrides for next the subsite in the queue and apply + overrides. Then generate the subsite using a PELICAN_CLASS + instance and its run method. Finally, restore the previous locale. + ''' + global _MAIN_SETTINGS + if len(_SUBSITE_QUEUE) == 0: + _LOGGER.debug( + 'i18n: Updating cross-site links and context of all generators.') + update_generators() + _MAIN_SETTINGS = None # to initialize next time + else: + with temporary_locale(): + settings = _MAIN_SETTINGS.copy() + lang, overrides = _SUBSITE_QUEUE.popitem() + settings.update(overrides) + settings = configure_settings(settings) # to set LOCALE, etc. + cls = get_pelican_cls(settings) + + new_pelican_obj = cls(settings) + _LOGGER.debug(("Generating i18n subsite for language '{}' " + "using class {}").format(lang, cls)) + new_pelican_obj.run() + + +# map: signal name -> function name +_SIGNAL_HANDLERS_DB = { + 'get_generators': initialize_plugin, + 'article_generator_pretaxonomy': filter_contents_translations, + 'page_generator_finalized': filter_contents_translations, + 'get_writer': create_next_subsite, + 'static_generator_finalized': save_main_static_files, + 'generator_init': save_generator, +} + def register(): - signals.initialized.connect(disable_lang_vars) - signals.generator_init.connect(install_templates_translations) - signals.article_generator_finalized.connect(update_generator_contents) - signals.page_generator_finalized.connect(update_generator_contents) - signals.finalized.connect(create_lang_subsites) + '''Register the plugin only if required signals are available''' + for sig_name in _SIGNAL_HANDLERS_DB.keys(): + if not hasattr(signals, sig_name): + _LOGGER.error(( + 'The i18n_subsites plugin requires the {} ' + 'signal available for sure in Pelican 3.4.0 and later, ' + 'plugin will not be used.').format(sig_name)) + return + + for sig_name, handler in _SIGNAL_HANDLERS_DB.items(): + sig = getattr(signals, sig_name) + sig.connect(handler) diff --git a/implementing_language_buttons.rst b/implementing_language_buttons.rst index 014cf1c..43f8bb3 100644 --- a/implementing_language_buttons.rst +++ b/implementing_language_buttons.rst @@ -2,9 +2,12 @@ Implementing language buttons ----------------------------- -Each article with translations has translations links, but that's the only way to switch between language subsites. +Each article with translations has translations links, but that's the +only way to switch between language subsites. -For this reason it is convenient to add language buttons to the top menu bar to make it simple to switch between the language subsites on all pages. +For this reason it is convenient to add language buttons to the top +menu bar to make it simple to switch between the language subsites on +all pages. Example designs --------------- @@ -12,7 +15,9 @@ Example designs Language buttons showing other available languages .................................................. -The ``extra_siteurls`` dictionary is a mapping of all other (not the *DEFAULT_LANG* of the current (sub-)site) languages to the *SITEURL* of the respective (sub-)sites +The ``extra_siteurls`` dictionary is a mapping of all other (not the +``DEFAULT_LANG`` of the current (sub-)site) languages to the +``SITEURL`` of the respective (sub-)sites .. code-block:: jinja @@ -20,7 +25,7 @@ The ``extra_siteurls`` dictionary is a mapping of all other (not the *DEFAULT_LA