From d7a4cc4a4a5ee3512bcb4918f235b8e5bf4a9703 Mon Sep 17 00:00:00 2001 From: Deepak Singhal Date: Wed, 24 Jul 2024 15:00:25 +0530 Subject: [PATCH 1/8] feat-3416: configurable-output-name-tags --- src/nominatim_api/localization.py | 44 ++++++++++++++++++++++++---- test/python/api/test_localization.py | 14 +++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/nominatim_api/localization.py b/src/nominatim_api/localization.py index 5964bbee8..f84e77f86 100644 --- a/src/nominatim_api/localization.py +++ b/src/nominatim_api/localization.py @@ -10,6 +10,8 @@ from typing import Mapping, List, Optional import re +import os + class Locales: """ Helper class for localization of names. @@ -22,11 +24,43 @@ def __init__(self, langs: Optional[List[str]] = None): self.languages = langs or [] self.name_tags: List[str] = [] - # Build the list of supported tags. It is currently hard-coded. - self._add_lang_tags('name') - self._add_tags('name', 'brand') - self._add_lang_tags('official_name', 'short_name') - self._add_tags('official_name', 'short_name', 'ref') + # Build the list of supported tags. + # It is now configurable with env setting NOMINATIM_OUTPUT_NAMES + # e.g. NOMINATIM_OUTPUT_NAMES = + # name:XX,name,brand,official_name:XX,short_name:XX,official_name,shirt_name,ref + nominatim_output_names = os.getenv("NOMINATIM_OUTPUT_NAMES") + if nominatim_output_names is None: + self._build_default_output_name_tags() + else: + self._build_output_name_tags(nominatim_output_names) + + + def _build_default_output_name_tags(self) -> None: + self._add_lang_tags("name") + self._add_tags("name", "brand") + self._add_lang_tags("official_name", "short_name") + self._add_tags("official_name", "short_name", "ref") + + + def _build_output_name_tags(self, nominatim_output_names: str) -> None: + nominatim_output_names = nominatim_output_names.split(",") + lang_tags = [] + tags = [] + for name in nominatim_output_names: + if name.endswith(":XX"): # Identifies Lang Tag + lang_tags.append(name[:-3]) + if tags:# Determines tags batch is over, lang tag batch begins + self._add_tags(*tags) + tags = [] + else: # Identifies Tag + tags.append(name) + if lang_tags: # Determines lang tags batch is over, tag batch begins + self._add_lang_tags(*lang_tags) + lang_tags = [] + if lang_tags: # Adds lefover lang tags + self._add_lang_tags(*lang_tags) + if tags: # Adds lefover tags + self._add_tags(*tags) def __bool__(self) -> bool: diff --git a/test/python/api/test_localization.py b/test/python/api/test_localization.py index 21fa72c81..2bdd2e136 100644 --- a/test/python/api/test_localization.py +++ b/test/python/api/test_localization.py @@ -7,6 +7,7 @@ """ Test functions for adapting results to the user's locale. """ +import os import pytest from nominatim_api import Locales @@ -51,3 +52,16 @@ def test_display_name_preference(): ('en,fr;garbage,de', ['en', 'de'])]) def test_from_language_preferences(langstr, langlist): assert Locales.from_accept_languages(langstr).languages == langlist + + +def test_configurable_output_name_tags(): + os.environ['NOMINATIM_OUTPUT_NAMES'] = 'name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref' + name_tags = {'name', '_place_name', 'brand', '_place_brand', 'official_name', '_place_official_name', 'short_name', '_place_short_name', 'ref', '_place_ref'} + l = Locales() + assert set(l.name_tags).difference(name_tags) == set() + + +def test_default_output_name_tags(): + name_tags = {'name', '_place_name', 'brand', '_place_brand', 'official_name', '_place_official_name', 'short_name', '_place_short_name', 'ref', '_place_ref'} + l = Locales() + assert set(l.name_tags).difference(name_tags) == set() \ No newline at end of file From 0352b59d2ef5c0157db53141db3acbad56709766 Mon Sep 17 00:00:00 2001 From: Deepak Singhal Date: Wed, 24 Jul 2024 15:31:39 +0530 Subject: [PATCH 2/8] docs-3416: docstrings and pylint --- test/python/api/test_localization.py | 38 +++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/test/python/api/test_localization.py b/test/python/api/test_localization.py index 2bdd2e136..5681b1ea6 100644 --- a/test/python/api/test_localization.py +++ b/test/python/api/test_localization.py @@ -55,13 +55,43 @@ def test_from_language_preferences(langstr, langlist): def test_configurable_output_name_tags(): - os.environ['NOMINATIM_OUTPUT_NAMES'] = 'name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref' - name_tags = {'name', '_place_name', 'brand', '_place_brand', 'official_name', '_place_official_name', 'short_name', '_place_short_name', 'ref', '_place_ref'} + """ + tests the output name tags when environment config is provided + """ + os.environ["NOMINATIM_OUTPUT_NAMES"] = ( + "name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref" + ) + name_tags = { + "name", + "_place_name", + "brand", + "_place_brand", + "official_name", + "_place_official_name", + "short_name", + "_place_short_name", + "ref", + "_place_ref", + } l = Locales() assert set(l.name_tags).difference(name_tags) == set() def test_default_output_name_tags(): - name_tags = {'name', '_place_name', 'brand', '_place_brand', 'official_name', '_place_official_name', 'short_name', '_place_short_name', 'ref', '_place_ref'} + """ + tests the default setting (previously hardcoded) in case environment config is not provided + """ + name_tags = { + "name", + "_place_name", + "brand", + "_place_brand", + "official_name", + "_place_official_name", + "short_name", + "_place_short_name", + "ref", + "_place_ref", + } l = Locales() - assert set(l.name_tags).difference(name_tags) == set() \ No newline at end of file + assert set(l.name_tags).difference(name_tags) == set() From b8ef4887b04dbd41b42dd7bda58ea41b734b0b0d Mon Sep 17 00:00:00 2001 From: Deepak Singhal Date: Wed, 24 Jul 2024 15:39:15 +0530 Subject: [PATCH 3/8] docs-3416: docstrings for implementation --- src/nominatim_api/localization.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/nominatim_api/localization.py b/src/nominatim_api/localization.py index f84e77f86..0da3bf641 100644 --- a/src/nominatim_api/localization.py +++ b/src/nominatim_api/localization.py @@ -25,9 +25,7 @@ def __init__(self, langs: Optional[List[str]] = None): self.name_tags: List[str] = [] # Build the list of supported tags. - # It is now configurable with env setting NOMINATIM_OUTPUT_NAMES - # e.g. NOMINATIM_OUTPUT_NAMES = - # name:XX,name,brand,official_name:XX,short_name:XX,official_name,shirt_name,ref + # Uses hard-coded when environment config not provided nominatim_output_names = os.getenv("NOMINATIM_OUTPUT_NAMES") if nominatim_output_names is None: self._build_default_output_name_tags() @@ -36,6 +34,9 @@ def __init__(self, langs: Optional[List[str]] = None): def _build_default_output_name_tags(self) -> None: + """ + Build the list of supported tags. Hard-coded implementation. + """ self._add_lang_tags("name") self._add_tags("name", "brand") self._add_lang_tags("official_name", "short_name") @@ -43,6 +44,16 @@ def _build_default_output_name_tags(self) -> None: def _build_output_name_tags(self, nominatim_output_names: str) -> None: + """ + Build the list of supported tags. Dynamic implementation. + Configurable through `NOMINATIM_OUTPUT_NAMES` environment config + Configuration input format: + `name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref` + - Coma separated + - `:XX` identifies lang tags + - consecutive `:XX` identifies multiple lang tags to-be added at the same time + - subsequent parts after lang tags (till next lang tag) identifies batch of tags + """ nominatim_output_names = nominatim_output_names.split(",") lang_tags = [] tags = [] From 22affcddcba498f39f1eec3f1148b7f5deb632ff Mon Sep 17 00:00:00 2001 From: Deepak Singhal Date: Fri, 26 Jul 2024 19:33:29 +0530 Subject: [PATCH 4/8] fix-3416: configuration using config class --- src/nominatim_api/localization.py | 36 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/nominatim_api/localization.py b/src/nominatim_api/localization.py index 0da3bf641..8c3d36428 100644 --- a/src/nominatim_api/localization.py +++ b/src/nominatim_api/localization.py @@ -7,30 +7,37 @@ """ Helper functions for localizing names of results. """ -from typing import Mapping, List, Optional - import re -import os +from typing import List, Mapping, Optional + +from .config import Configuration class Locales: """ Helper class for localization of names. - It takes a list of language prefixes in their order of preferred - usage. + It takes a list of language prefixes in their order of preferred + usage. + It takes config object as input which enables to configure the + list of supported tags. Setting name: `OUTPUT_NAMES` + How to use? + e.g. `name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref` + - Coma separated + - `:XX` identifies lang tags + - consecutive `:XX` identifies multiple lang tags to-be added at the same time + - subsequent parts after lang tags (till next lang tag) identifies batch of tags """ - def __init__(self, langs: Optional[List[str]] = None): + def __init__(self, langs: Optional[List[str]] = None, config: Optional[Configuration] = None): self.languages = langs or [] self.name_tags: List[str] = [] - # Build the list of supported tags. - # Uses hard-coded when environment config not provided - nominatim_output_names = os.getenv("NOMINATIM_OUTPUT_NAMES") - if nominatim_output_names is None: - self._build_default_output_name_tags() + # Build the list of supported tags + # Uses hard-coded when config `OUTPUT_NAMES` not provided + if config and config.OUTPUT_NAMES != '': + self._build_output_name_tags(config.OUTPUT_NAMES) else: - self._build_output_name_tags(nominatim_output_names) + self._build_default_output_name_tags() def _build_default_output_name_tags(self) -> None: @@ -46,9 +53,8 @@ def _build_default_output_name_tags(self) -> None: def _build_output_name_tags(self, nominatim_output_names: str) -> None: """ Build the list of supported tags. Dynamic implementation. - Configurable through `NOMINATIM_OUTPUT_NAMES` environment config - Configuration input format: - `name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref` + How to use? + e.g. `name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref` - Coma separated - `:XX` identifies lang tags - consecutive `:XX` identifies multiple lang tags to-be added at the same time From ab4e42439e4982b926b68eb193e5b802f6bcad40 Mon Sep 17 00:00:00 2001 From: Deepak Singhal Date: Fri, 26 Jul 2024 19:47:36 +0530 Subject: [PATCH 5/8] fix-3416: updated test cases with configuration --- test/python/api/test_localization.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/python/api/test_localization.py b/test/python/api/test_localization.py index 5681b1ea6..fed0bb177 100644 --- a/test/python/api/test_localization.py +++ b/test/python/api/test_localization.py @@ -10,7 +10,7 @@ import os import pytest -from nominatim_api import Locales +from nominatim_api import Locales, Configuration def test_display_name_empty_names(): l = Locales(['en', 'de']) @@ -58,9 +58,6 @@ def test_configurable_output_name_tags(): """ tests the output name tags when environment config is provided """ - os.environ["NOMINATIM_OUTPUT_NAMES"] = ( - "name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref" - ) name_tags = { "name", "_place_name", @@ -68,12 +65,19 @@ def test_configurable_output_name_tags(): "_place_brand", "official_name", "_place_official_name", - "short_name", - "_place_short_name", + "shirt_name", + "_place_shirt_name", "ref", "_place_ref", } - l = Locales() + for name in list(name_tags): name_tags.add(f"{name}:en") + cfg = Configuration( + project_dir=None, + environ={ + "NOMINATIM_OUTPUT_NAMES":"name:XX,name,brand,official_name:XX,shirt_name:XX,official_name,shirt_name,ref" + } + ) + l = Locales(['en'], config=cfg) assert set(l.name_tags).difference(name_tags) == set() From e0a671ab2c748d4c2a283356ed5cefdc0865adc6 Mon Sep 17 00:00:00 2001 From: Deepak Singhal Date: Wed, 31 Jul 2024 16:01:36 +0530 Subject: [PATCH 6/8] refactor-3416: review changes --- src/nominatim_api/localization.py | 57 ++++++++++------------------ test/python/api/test_localization.py | 33 ++++++++-------- 2 files changed, 37 insertions(+), 53 deletions(-) diff --git a/src/nominatim_api/localization.py b/src/nominatim_api/localization.py index 8c3d36428..819991a04 100644 --- a/src/nominatim_api/localization.py +++ b/src/nominatim_api/localization.py @@ -7,62 +7,43 @@ """ Helper functions for localizing names of results. """ -import re from typing import List, Mapping, Optional - -from .config import Configuration +import re class Locales: """ Helper class for localization of names. - It takes a list of language prefixes in their order of preferred - usage. - It takes config object as input which enables to configure the - list of supported tags. Setting name: `OUTPUT_NAMES` - How to use? - e.g. `name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref` - - Coma separated - - `:XX` identifies lang tags - - consecutive `:XX` identifies multiple lang tags to-be added at the same time - - subsequent parts after lang tags (till next lang tag) identifies batch of tags + Keyword arguments: + langs: List[str] -- list of language prefixes in + their order of preferred usage. + output_name_tags_config: str -- string object + containing output name tags in their order of preferred usage. + default -- + `name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref` """ - def __init__(self, langs: Optional[List[str]] = None, config: Optional[Configuration] = None): + def __init__( + self, + langs: Optional[List[str]] = None, + output_name_tags_config: str = ( + "name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref" + ) + ): self.languages = langs or [] self.name_tags: List[str] = [] # Build the list of supported tags - # Uses hard-coded when config `OUTPUT_NAMES` not provided - if config and config.OUTPUT_NAMES != '': - self._build_output_name_tags(config.OUTPUT_NAMES) - else: - self._build_default_output_name_tags() - - - def _build_default_output_name_tags(self) -> None: - """ - Build the list of supported tags. Hard-coded implementation. - """ - self._add_lang_tags("name") - self._add_tags("name", "brand") - self._add_lang_tags("official_name", "short_name") - self._add_tags("official_name", "short_name", "ref") + self._build_output_name_tags(output_name_tags_config) def _build_output_name_tags(self, nominatim_output_names: str) -> None: """ Build the list of supported tags. Dynamic implementation. - How to use? - e.g. `name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref` - - Coma separated - - `:XX` identifies lang tags - - consecutive `:XX` identifies multiple lang tags to-be added at the same time - - subsequent parts after lang tags (till next lang tag) identifies batch of tags """ - nominatim_output_names = nominatim_output_names.split(",") - lang_tags = [] - tags = [] + nominatim_output_names: List[str] = nominatim_output_names.split(",") + lang_tags: List[str] = [] + tags: List[str] = [] for name in nominatim_output_names: if name.endswith(":XX"): # Identifies Lang Tag lang_tags.append(name[:-3]) diff --git a/test/python/api/test_localization.py b/test/python/api/test_localization.py index fed0bb177..59756929e 100644 --- a/test/python/api/test_localization.py +++ b/test/python/api/test_localization.py @@ -7,10 +7,9 @@ """ Test functions for adapting results to the user's locale. """ -import os import pytest -from nominatim_api import Locales, Configuration +from nominatim_api import Locales def test_display_name_empty_names(): l = Locales(['en', 'de']) @@ -58,34 +57,38 @@ def test_configurable_output_name_tags(): """ tests the output name tags when environment config is provided """ - name_tags = { + name_tags = [ + "name:en", + "_place_name:en", "name", "_place_name", "brand", "_place_brand", + "official_name:en", + "_place_official_name:en", + "shirt_name:en", + "_place_shirt_name:en", "official_name", "_place_official_name", "shirt_name", "_place_shirt_name", "ref", "_place_ref", - } - for name in list(name_tags): name_tags.add(f"{name}:en") - cfg = Configuration( - project_dir=None, - environ={ - "NOMINATIM_OUTPUT_NAMES":"name:XX,name,brand,official_name:XX,shirt_name:XX,official_name,shirt_name,ref" - } + ] + l = Locales( + ['en'], + output_name_tags_config=( + "name:XX,name,brand,official_name:XX,shirt_name:XX,official_name,shirt_name,ref" + ) ) - l = Locales(['en'], config=cfg) - assert set(l.name_tags).difference(name_tags) == set() + assert l.name_tags == name_tags def test_default_output_name_tags(): """ tests the default setting (previously hardcoded) in case environment config is not provided """ - name_tags = { + name_tags = [ "name", "_place_name", "brand", @@ -96,6 +99,6 @@ def test_default_output_name_tags(): "_place_short_name", "ref", "_place_ref", - } + ] l = Locales() - assert set(l.name_tags).difference(name_tags) == set() + assert l.name_tags == name_tags From b0f8d6ab2760fc795a63cd2182eb960fbec04596 Mon Sep 17 00:00:00 2001 From: Deepak Singhal Date: Wed, 31 Jul 2024 16:40:46 +0530 Subject: [PATCH 7/8] docs-3416: added the configuration to env defaults and docs --- docs/customize/Settings.md | 19 +++++++++++++++++++ settings/env.defaults | 6 ++++++ src/nominatim_api/localization.py | 1 + 3 files changed, 26 insertions(+) diff --git a/docs/customize/Settings.md b/docs/customize/Settings.md index 8245e309f..157076fde 100644 --- a/docs/customize/Settings.md +++ b/docs/customize/Settings.md @@ -689,6 +689,25 @@ results gathered so far. Note that under high load you may observe that users receive different results than usual without seeing an error. This may cause some confusion. + +#### NOMINATIM_OUTPUT_NAMES + +| Summary | | +| -------------- | --------------------------------------------------- | +| **Description:** | Determines which name tags is chosen for a place | +| **Format:** | string | +| **Default:** | name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref | +| **Comment:** | Python frontend only | + + +Accepts Coma separated values, `:XX` at the end identifies language specific tags +else value is identified as general tag. Tags will be added based on the given order. + +Take `name:XX,name,brand` for example: +first name will added as language tag for given languages, +then name and brand will added as place name tags + + ### Logging Settings #### NOMINATIM_LOG_DB diff --git a/settings/env.defaults b/settings/env.defaults index f4c33e772..20aa9734c 100644 --- a/settings/env.defaults +++ b/settings/env.defaults @@ -220,6 +220,12 @@ NOMINATIM_REQUEST_TIMEOUT=60 # to geocode" instead. NOMINATIM_SEARCH_WITHIN_COUNTRIES=False +# Determines which name tags is chosen for a place. +# Accepts Coma separated values, `:XX` at the end identifies +# language specific tags else value is identified as general tag. +# Tags will be added based on the given order. +NOMINATIM_OUTPUT_NAMES=name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref + ### Log settings # # The following options allow to enable logging of API requests. diff --git a/src/nominatim_api/localization.py b/src/nominatim_api/localization.py index 819991a04..2387fed08 100644 --- a/src/nominatim_api/localization.py +++ b/src/nominatim_api/localization.py @@ -8,6 +8,7 @@ Helper functions for localizing names of results. """ from typing import List, Mapping, Optional + import re From 9cd84bf87cd915c02aa4c298c73522fecb09aca7 Mon Sep 17 00:00:00 2001 From: Deepak Singhal Date: Wed, 31 Jul 2024 16:45:59 +0530 Subject: [PATCH 8/8] refactor-3416: revert import order --- src/nominatim_api/localization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nominatim_api/localization.py b/src/nominatim_api/localization.py index 2387fed08..0d96c76c1 100644 --- a/src/nominatim_api/localization.py +++ b/src/nominatim_api/localization.py @@ -7,7 +7,7 @@ """ Helper functions for localizing names of results. """ -from typing import List, Mapping, Optional +from typing import Mapping, List, Optional import re