From 4a1c78f82368123a92547051ecb073f1bdbce705 Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Fri, 14 Jul 2023 03:33:14 +0100
Subject: [PATCH 1/5] feat/pipeline

---
 mycroft/skills/intent_services/__init__.py    |  10 +-
 .../intent_services/padatious_service.py      |   2 +-
 ovos_core/intent_services/__init__.py         | 116 ++++----
 .../intent_services/padacioso_service.py      | 248 ++++++++++++++++++
 .../intent_services/padatious_service.py      | 122 ++-------
 .../skills/test_utterance_intents.py          |  77 +++---
 6 files changed, 393 insertions(+), 182 deletions(-)
 create mode 100644 ovos_core/intent_services/padacioso_service.py

diff --git a/mycroft/skills/intent_services/__init__.py b/mycroft/skills/intent_services/__init__.py
index 3ae3dc6b24a4..c8c306e139df 100644
--- a/mycroft/skills/intent_services/__init__.py
+++ b/mycroft/skills/intent_services/__init__.py
@@ -1,8 +1,14 @@
 from ovos_core.intent_services import AdaptService,\
-    PadatiousService, PadatiousMatcher, \
     ConverseService,\
     CommonQAService, \
-    FallbackService
+    FallbackService, \
+    PadaciosoService, \
+    PadatiousService
 from ovos_core.intent_services import IntentMatch
 from mycroft.skills.intent_services.adapt_service import AdaptIntent, IntentBuilder, Intent
 
+try:  # TODO -remove backwards compat import, before 0.0.8, ovos_core module didnt make it into a stable release yet!
+    from ovos_core.intent_services import PadatiousMatcher
+except ImportError:
+    from ovos_utils.log import LOG
+    LOG.warning("padatious not installed")
diff --git a/mycroft/skills/intent_services/padatious_service.py b/mycroft/skills/intent_services/padatious_service.py
index 2751df892bd2..b5d1809a4312 100644
--- a/mycroft/skills/intent_services/padatious_service.py
+++ b/mycroft/skills/intent_services/padatious_service.py
@@ -13,5 +13,5 @@
 # limitations under the License.
 #
 """Intent service wrapping padatious."""
-from ovos_core.intent_services.padatious_service import PadatiousMatcher, PadatiousService, PadatiousIntent, FallbackIntentContainer
+from ovos_core.intent_services.padatious_service import PadatiousMatcher, PadatiousService, PadatiousIntent
 
diff --git a/ovos_core/intent_services/__init__.py b/ovos_core/intent_services/__init__.py
index 8b2e54b3b04f..fdad982571fb 100644
--- a/ovos_core/intent_services/__init__.py
+++ b/ovos_core/intent_services/__init__.py
@@ -16,19 +16,24 @@
 
 from ovos_config.config import Configuration
 from ovos_config.locale import setup_locale
-
-from ovos_core.transformers import MetadataTransformersService, UtteranceTransformersService
-from ovos_core.intent_services.adapt_service import AdaptService
-from ovos_core.intent_services.commonqa_service import CommonQAService
-from ovos_core.intent_services.converse_service import ConverseService
-from ovos_core.intent_services.fallback_service import FallbackService
-from ovos_core.intent_services.padatious_service import PadatiousService, PadatiousMatcher
 from ovos_utils.intents.intent_service_interface import open_intent_envelope
 from ovos_utils.log import LOG
 from ovos_utils.messagebus import get_message_lang
 from ovos_utils.metrics import Stopwatch
 from ovos_utils.sound import play_error_sound
 
+from ovos_core.intent_services.adapt_service import AdaptService
+from ovos_core.intent_services.commonqa_service import CommonQAService
+from ovos_core.intent_services.converse_service import ConverseService
+from ovos_core.intent_services.fallback_service import FallbackService
+from ovos_core.intent_services.padacioso_service import PadaciosoService
+from ovos_core.transformers import MetadataTransformersService, UtteranceTransformersService
+
+try:
+    from ovos_core.intent_services.padatious_service import PadatiousService, PadatiousMatcher
+except ImportError:
+    from ovos_core.intent_services.padacioso_service import PadaciosoService as PadatiousService
+
 # Intent match response tuple containing
 # intent_service: Name of the service that matched the intent
 # intent_type: intent name (used to call intent handler over the message bus)
@@ -56,10 +61,12 @@ def __init__(self, bus):
 
         # TODO - replace with plugins
         self.adapt_service = AdaptService(config.get('context', {}))
-        try:
+        if PadaciosoService is not PadatiousService:
             self.padatious_service = PadatiousService(bus, config['padatious'])
-        except Exception as err:
-            LOG.exception(f'Failed to create padatious handlers ({err})')
+        else:
+            LOG.error(f'Failed to create padatious handlers, padatious not installed')
+            self.padatious_service = None
+        self.padacioso_service = PadaciosoService(bus, config['padatious'])
         self.fallback = FallbackService(bus)
         self.converse = ConverseService(bus)
         self.common_qa = CommonQAService(bus)
@@ -105,6 +112,22 @@ def __init__(self, bus):
         self.bus.on('intent.service.padatious.entities.manifest.get',
                     self.handle_entity_manifest)
 
+    @property
+    def pipeline(self):
+        # List of functions to use to match the utterance with intent, listed in priority order.
+        config = Configuration().get("intents") or {}
+        return config.get("pipeline", [
+            "converse",
+            "padacioso_high",
+            "adapt",
+            "common_qa",
+            "fallback_high",
+            "padacioso_medium",
+            "fallback_medium",
+            "padacioso_low",
+            "fallback_low"
+        ])
+
     @property
     def registered_intents(self):
         lang = get_message_lang()
@@ -199,6 +222,38 @@ def disambiguate_lang(message):
 
         return default_lang
 
+    def get_pipeline(self, skip_converse=False, skip_fallback=False):
+        # Create matchers
+        # TODO - from plugins
+        if self.padatious_service is None:
+            if any("padatious" in p for p in self.pipeline):
+                LOG.warning("padatious is not available! using padacioso in it's place")
+            padatious_matcher = self.padacioso_service
+        else:
+            from ovos_core.intent_services.padatious_service import PadatiousMatcher
+            padatious_matcher = PadatiousMatcher(self.padatious_service)
+
+        matchers = {
+            "converse": self.converse.converse_with_skills,
+            "padatious_high": padatious_matcher.match_high,
+            "padacioso_high": self.padacioso_service.match_high,
+            "adapt": self.adapt_service.match_intent,
+            "common_qa": self.common_qa.match,
+            "fallback_high": self.fallback.high_prio,
+            "padatious_medium": padatious_matcher.match_medium,
+            "padacioso_medium": self.padacioso_service.match_medium,
+            "fallback_medium": self.fallback.medium_prio,
+            "padatious_low": padatious_matcher.match_low,
+            "padacioso_low": self.padacioso_service.match_low,
+            "fallback_low": self.fallback.low_prio
+        }
+        pipeline = list(self.pipeline)
+        if skip_converse and "converse" in pipeline:
+            pipeline.remove("converse")
+        if skip_fallback:
+            pipeline = [p for p in pipeline if not p.startswith("fallback_")]
+        return [matchers[k] for k in pipeline]
+
     def handle_utterance(self, message):
         """Main entrypoint for handling user utterances
 
@@ -242,24 +297,11 @@ def handle_utterance(self, message):
 
             stopwatch = Stopwatch()
 
-            # Create matchers
-            padatious_matcher = PadatiousMatcher(self.padatious_service)
-
-            # List of functions to use to match the utterance with intent.
-            # These are listed in priority order.
-            match_funcs = [
-                self.converse.converse_with_skills, padatious_matcher.match_high,
-                self.adapt_service.match_intent, self.common_qa.match,
-                self.fallback.high_prio, padatious_matcher.match_medium,
-                self.fallback.medium_prio, padatious_matcher.match_low,
-                self.fallback.low_prio
-            ]
-
             # match
             match = None
             with stopwatch:
                 # Loop through the matching functions until a match is found.
-                for match_func in match_funcs:
+                for match_func in self.get_pipeline():
                     match = match_func(utterances, lang, message)
                     if match:
                         break
@@ -392,24 +434,8 @@ def handle_get_intent(self, message):
         utterance = message.data["utterance"]
         lang = get_message_lang(message)
 
-        # Create matchers
-        padatious_matcher = PadatiousMatcher(self.padatious_service)
-
-        # List of functions to use to match the utterance with intent.
-        # These are listed in priority order.
-        # TODO once we have a mechanism for checking if a fallback will
-        #  trigger without actually triggering it, those should be added here
-        match_funcs = [
-            padatious_matcher.match_high,
-            self.adapt_service.match_intent,
-            # self.fallback.high_prio,
-            padatious_matcher.match_medium,
-            # self.fallback.medium_prio,
-            padatious_matcher.match_low,
-            # self.fallback.low_prio
-        ]
         # Loop through the matching functions until a match is found.
-        for match_func in match_funcs:
+        for match_func in self.get_pipeline(skip_converse=True, skip_fallback=True):
             match = match_func([utterance], lang, message)
             if match:
                 if match.intent_type:
@@ -483,9 +509,9 @@ def handle_get_padatious(self, message):
         """
         utterance = message.data["utterance"]
         norm = message.data.get('norm_utt', utterance)
-        intent = self.padatious_service.calc_intent(utterance)
+        intent = self.padacioso_service.calc_intent(utterance)
         if not intent and norm != utterance:
-            intent = self.padatious_service.calc_intent(norm)
+            intent = self.padacioso_service.calc_intent(norm)
         if intent:
             intent = intent.__dict__
         self.bus.emit(message.reply("intent.service.padatious.reply",
@@ -499,7 +525,7 @@ def handle_padatious_manifest(self, message):
         """
         self.bus.emit(message.reply(
             "intent.service.padatious.manifest",
-            {"intents": self.padatious_service.registered_intents}))
+            {"intents": self.padacioso_service.registered_intents}))
 
     def handle_entity_manifest(self, message):
         """Messagebus handler returning the registered padatious entities.
@@ -509,7 +535,7 @@ def handle_entity_manifest(self, message):
         """
         self.bus.emit(message.reply(
             "intent.service.padatious.entities.manifest",
-            {"entities": self.padatious_service.registered_entities}))
+            {"entities": self.padacioso_service.registered_entities}))
 
 
 def _is_old_style_keyword_message(message):
diff --git a/ovos_core/intent_services/padacioso_service.py b/ovos_core/intent_services/padacioso_service.py
new file mode 100644
index 000000000000..a34e2e0d4555
--- /dev/null
+++ b/ovos_core/intent_services/padacioso_service.py
@@ -0,0 +1,248 @@
+"""Intent service wrapping padacioso."""
+import concurrent.futures
+from functools import lru_cache
+from os.path import isfile
+from typing import List, Optional
+
+from ovos_config.config import Configuration
+from ovos_utils import flatten_list
+from ovos_utils.log import LOG
+from padacioso import IntentContainer as FallbackIntentContainer
+
+import ovos_core.intent_services
+from ovos_bus_client.message import Message
+
+
+class PadaciosoIntent:
+    """
+    A set of data describing how a query fits into an intent
+    Attributes:
+        name (str): Name of matched intent
+        sent (str): The input utterance associated with the intent
+        conf (float): Confidence (from 0.0 to 1.0)
+        matches (dict of str -> str): Key is the name of the entity and
+            value is the extracted part of the sentence
+    """
+
+    def __init__(self, name, sent, matches=None, conf=0.0):
+        self.name = name
+        self.sent = sent
+        self.matches = matches or {}
+        self.conf = conf
+
+    def __getitem__(self, item):
+        return self.matches.__getitem__(item)
+
+    def __contains__(self, item):
+        return self.matches.__contains__(item)
+
+    def get(self, key, default=None):
+        return self.matches.get(key, default)
+
+    def __repr__(self):
+        return repr(self.__dict__)
+
+
+class PadaciosoService:
+    """Service class for padacioso intent matching."""
+
+    def __init__(self, bus, config):
+        self.padacioso_config = config
+        self.bus = bus
+
+        core_config = Configuration()
+        self.lang = core_config.get("lang", "en-us")
+        langs = core_config.get('secondary_langs') or []
+        if self.lang not in langs:
+            langs.append(self.lang)
+
+        self.conf_high = self.padacioso_config.get("conf_high") or 0.95
+        self.conf_med = self.padacioso_config.get("conf_med") or 0.8
+        self.conf_low = self.padacioso_config.get("conf_low") or 0.5
+        self.workers = self.padacioso_config.get("workers") or 4
+
+        LOG.debug('Using Padacioso intent parser.')
+        self.containers = {lang: FallbackIntentContainer(
+            self.padacioso_config.get("fuzz"), n_workers=self.workers)
+            for lang in langs}
+
+        self.bus.on('padatious:register_intent', self.register_intent)
+        self.bus.on('padatious:register_entity', self.register_entity)
+        self.bus.on('detach_intent', self.handle_detach_intent)
+        self.bus.on('detach_skill', self.handle_detach_skill)
+
+        self.registered_intents = []
+        self.registered_entities = []
+
+    def _match_level(self, utterances, limit, lang=None):
+        """Match intent and make sure a certain level of confidence is reached.
+
+        Args:
+            utterances (list of tuples): Utterances to parse, originals paired
+                                         with optional normalized version.
+            limit (float): required confidence level.
+        """
+        LOG.debug(f'Padacioso Matching confidence > {limit}')
+        # call flatten in case someone is sending the old style list of tuples
+        utterances = flatten_list(utterances)
+        lang = lang or self.lang
+        padacioso_intent = self.calc_intent(utterances, lang)
+        if padacioso_intent is not None and padacioso_intent.conf > limit:
+            skill_id = padacioso_intent.name.split(':')[0]
+            return ovos_core.intent_services.IntentMatch(
+                'Padacioso', padacioso_intent.name,
+                padacioso_intent.matches, skill_id, padacioso_intent.sent)
+
+    def match_high(self, utterances, lang=None, message=None):
+        """Intent matcher for high confidence.
+
+        Args:
+            utterances (list of tuples): Utterances to parse, originals paired
+                                         with optional normalized version.
+        """
+        return self._match_level(utterances, self.conf_high, lang)
+
+    def match_medium(self, utterances, lang=None, message=None):
+        """Intent matcher for medium confidence.
+
+        Args:
+            utterances (list of tuples): Utterances to parse, originals paired
+                                         with optional normalized version.
+        """
+        return self._match_level(utterances, self.conf_med, lang)
+
+    def match_low(self, utterances, lang=None, message=None):
+        """Intent matcher for low confidence.
+
+        Args:
+            utterances (list of tuples): Utterances to parse, originals paired
+                                         with optional normalized version.
+        """
+        return self._match_level(utterances, self.conf_low, lang)
+
+    def __detach_intent(self, intent_name):
+        """ Remove an intent if it has been registered.
+
+        Args:
+            intent_name (str): intent identifier
+        """
+        if intent_name in self.registered_intents:
+            self.registered_intents.remove(intent_name)
+            for lang in self.containers:
+                self.containers[lang].remove_intent(intent_name)
+
+    def handle_detach_intent(self, message):
+        """Messagebus handler for detaching padacioso intent.
+
+        Args:
+            message (Message): message triggering action
+        """
+        self.__detach_intent(message.data.get('intent_name'))
+
+    def handle_detach_skill(self, message):
+        """Messagebus handler for detaching all intents for skill.
+
+        Args:
+            message (Message): message triggering action
+        """
+        skill_id = message.data['skill_id']
+        remove_list = [i for i in self.registered_intents if skill_id in i]
+        for i in remove_list:
+            self.__detach_intent(i)
+
+    def _register_object(self, message, object_name, register_func):
+        """Generic method for registering a padacioso object.
+
+        Args:
+            message (Message): trigger for action
+            object_name (str): type of entry to register
+            register_func (callable): function to call for registration
+        """
+        file_name = message.data.get('file_name')
+        samples = message.data.get("samples")
+        name = message.data['name']
+
+        LOG.debug('Registering Padacioso ' + object_name + ': ' + name)
+
+        if (not file_name or not isfile(file_name)) and not samples:
+            LOG.error('Could not find file ' + file_name)
+            return
+
+        if not samples and isfile(file_name):
+            with open(file_name) as f:
+                samples = [l.strip() for l in f.readlines()]
+
+        register_func(name, samples)
+
+    def register_intent(self, message):
+        """Messagebus handler for registering intents.
+
+        Args:
+            message (Message): message triggering action
+        """
+        lang = message.data.get('lang', self.lang)
+        lang = lang.lower()
+        if lang in self.containers:
+            self.registered_intents.append(message.data['name'])
+            try:
+                self._register_object(message, 'intent',
+                                      self.containers[lang].add_intent)
+            except RuntimeError:
+                name = message.data.get('name', "")
+                # padacioso fails on reloading a skill, just ignore
+                if name not in self.containers[lang].intent_samples:
+                    raise
+
+    def register_entity(self, message):
+        """Messagebus handler for registering entities.
+
+        Args:
+            message (Message): message triggering action
+        """
+        lang = message.data.get('lang', self.lang)
+        lang = lang.lower()
+        if lang in self.containers:
+            self.registered_entities.append(message.data)
+            self._register_object(message, 'entity',
+                                  self.containers[lang].add_entity)
+
+    def calc_intent(self, utterances: List[str], lang: str = None) -> Optional[PadaciosoIntent]:
+        """
+        Get the best intent match for the given list of utterances. Utilizes a
+        thread pool for overall faster execution. Note that this method is NOT
+        compatible with Padacioso, but is compatible with Padacioso.
+        @param utterances: list of string utterances to get an intent for
+        @param lang: language of utterances
+        @return:
+        """
+        if isinstance(utterances, str):
+            utterances = [utterances]  # backwards compat when arg was a single string
+        lang = lang or self.lang
+        lang = lang.lower()
+        if lang in self.containers:
+            intent_container = self.containers.get(lang)
+            intents = [_calc_padacioso_intent(utt, intent_container) for utt in utterances]
+            intents = [i for i in intents if i is not None]
+            # select best
+            if intents:
+                return max(intents, key=lambda k: k.conf)
+
+
+@lru_cache(maxsize=3)  # repeat calls under different conf levels wont re-run code
+def _calc_padacioso_intent(utt, intent_container) -> \
+        Optional[PadaciosoIntent]:
+    """
+    Try to match an utterance to an intent in an intent_container
+    @param args: tuple of (utterance, IntentContainer)
+    @return: matched PadaciosoIntent
+    """
+    try:
+        intent = intent_container.calc_intent(utt)
+        if "entities" in intent:
+            intent["matches"] = intent.pop("entities")
+        intent["sent"] = utt
+        intent = PadaciosoIntent(**intent)
+        intent.sent = utt
+        return intent
+    except Exception as e:
+        LOG.error(e)
diff --git a/ovos_core/intent_services/padatious_service.py b/ovos_core/intent_services/padatious_service.py
index e6ef7ef372e0..3c0e83112147 100644
--- a/ovos_core/intent_services/padatious_service.py
+++ b/ovos_core/intent_services/padatious_service.py
@@ -21,51 +21,17 @@
 from time import time as get_time, sleep
 from typing import List, Optional
 
+import padatious
+from padatious.match_data import MatchData as PadatiousIntent
 from ovos_config.config import Configuration
 from ovos_config.meta import get_xdg_base
 from ovos_utils import flatten_list
 from ovos_utils.log import LOG
 from ovos_utils.xdg_utils import xdg_data_home
-from padacioso import IntentContainer as FallbackIntentContainer
 
 import ovos_core.intent_services
 from ovos_bus_client.message import Message
 
-try:
-    import padatious as _pd
-except ImportError:
-    _pd = None
-
-
-class PadatiousIntent:
-    """
-    A set of data describing how a query fits into an intent
-    Attributes:
-        name (str): Name of matched intent
-        sent (str): The input utterance associated with the intent
-        conf (float): Confidence (from 0.0 to 1.0)
-        matches (dict of str -> str): Key is the name of the entity and
-            value is the extracted part of the sentence
-    """
-
-    def __init__(self, name, sent, matches=None, conf=0.0):
-        self.name = name
-        self.sent = sent
-        self.matches = matches or {}
-        self.conf = conf
-
-    def __getitem__(self, item):
-        return self.matches.__getitem__(item)
-
-    def __contains__(self, item):
-        return self.matches.__contains__(item)
-
-    def get(self, key, default=None):
-        return self.matches.get(key, default)
-
-    def __repr__(self):
-        return repr(self.__dict__)
-
 
 class PadatiousMatcher:
     """Matcher class to avoid redundancy in padatious intent matching."""
@@ -81,10 +47,7 @@ def _match_level(self, utterances, limit, lang=None):
                                          with optional normalized version.
             limit (float): required confidence level.
         """
-        if self.service.is_regex_only:
-            LOG.debug(f'Padacioso Matching confidence > {limit}')
-        else:
-            LOG.debug(f'Padatious Matching confidence > {limit}')
+        LOG.debug(f'Padatious Matching confidence > {limit}')
         # call flatten in case someone is sending the old style list of tuples
         utterances = flatten_list(utterances)
         lang = lang or self.service.lang
@@ -139,20 +102,13 @@ def __init__(self, bus, config):
         self.conf_high = self.padatious_config.get("conf_high") or 0.95
         self.conf_med = self.padatious_config.get("conf_med") or 0.8
         self.conf_low = self.padatious_config.get("conf_low") or 0.5
-        self.workers = self.padatious_config.get("workers") or 4
 
-        if self.is_regex_only:
-            LOG.debug('Using Padacioso intent parser.')
-            self.containers = {lang: FallbackIntentContainer(
-                self.padatious_config.get("fuzz"), n_workers=self.workers)
-                for lang in langs}
-        else:
-            LOG.debug('Using Padatious intent parser.')
-            intent_cache = self.padatious_config.get(
-                'intent_cache') or f"{xdg_data_home()}/{get_xdg_base()}/intent_cache"
-            self.containers = {
-                lang: _pd.IntentContainer(path.join(expanduser(intent_cache), lang))
-                for lang in langs}
+        LOG.debug('Using Padatious intent parser.')
+        intent_cache = self.padatious_config.get(
+            'intent_cache') or f"{xdg_data_home()}/{get_xdg_base()}/intent_cache"
+        self.containers = {
+            lang: padatious.IntentContainer(path.join(expanduser(intent_cache), lang))
+            for lang in langs}
 
         self.bus.on('padatious:register_intent', self.register_intent)
         self.bus.on('padatious:register_entity', self.register_entity)
@@ -169,19 +125,6 @@ def __init__(self, bus, config):
         self.registered_intents = []
         self.registered_entities = []
 
-    @property
-    def is_regex_only(self):
-        if not _pd:
-            return True
-        return self.padatious_config.get("regex_only") or False
-
-    @property
-    def threaded_inference(self):
-        LOG.warning("threaded_inference config has been deprecated")
-        # Padatious isn't thread-safe, so don't even try
-        # Padacioso already uses concurrent.futures internally, no benefit
-        return False
-
     def train(self, message=None):
         """Perform padatious training.
 
@@ -189,15 +132,14 @@ def train(self, message=None):
             message (Message): optional triggering message
         """
         self.finished_training_event.clear()
-        if not self.is_regex_only:
-            padatious_single_thread = self.padatious_config.get('single_thread', True)
-            if message is None:
-                single_thread = padatious_single_thread
-            else:
-                single_thread = message.data.get('single_thread',
-                                                 padatious_single_thread)
-            for lang in self.containers:
-                self.containers[lang].train(single_thread=single_thread)
+        padatious_single_thread = self.padatious_config.get('single_thread', True)
+        if message is None:
+            single_thread = padatious_single_thread
+        else:
+            single_thread = message.data.get('single_thread',
+                                             padatious_single_thread)
+        for lang in self.containers:
+            self.containers[lang].train(single_thread=single_thread)
 
         LOG.info('Training complete.')
         self.finished_training_event.set()
@@ -271,9 +213,8 @@ def _register_object(self, message, object_name, register_func):
 
         register_func(name, samples)
 
-        if not self.is_regex_only:
-            self.train_time = get_time() + self.train_delay
-            self.wait_and_train()
+        self.train_time = get_time() + self.train_delay
+        self.wait_and_train()
 
     def register_intent(self, message):
         """Messagebus handler for registering intents.
@@ -285,8 +226,14 @@ def register_intent(self, message):
         lang = lang.lower()
         if lang in self.containers:
             self.registered_intents.append(message.data['name'])
-            self._register_object(message, 'intent',
-                                  self.containers[lang].add_intent)
+            try:
+                self._register_object(message, 'intent',
+                                      self.containers[lang].add_intent)
+            except RuntimeError:
+                name = message.data.get('name', "")
+                # padacioso fails on reloading a skill, just ignore
+                if name not in self.containers[lang].intent_samples:
+                    raise
 
     def register_entity(self, message):
         """Messagebus handler for registering entities.
@@ -323,26 +270,15 @@ def calc_intent(self, utterances: List[str], lang: str = None) -> Optional[Padat
                 return max(intents, key=lambda k: k.conf)
 
 
-@lru_cache(maxsize=10)  # repeat calls under different conf levels wont re-run code
-def _calc_padatious_intent(*args) -> \
-        Optional[PadatiousIntent]:
+@lru_cache(maxsize=3)  # repeat calls under different conf levels wont re-run code
+def _calc_padatious_intent(utt, intent_container) -> Optional[PadatiousIntent]:
     """
     Try to match an utterance to an intent in an intent_container
     @param args: tuple of (utterance, IntentContainer)
     @return: matched PadatiousIntent
     """
     try:
-        if len(args) == 1:
-            args = args[0]
-        utt = args[0]
-        intent_container = args[1]
         intent = intent_container.calc_intent(utt)
-        if isinstance(intent, dict):
-            if "entities" in intent:
-                intent["matches"] = intent.pop("entities")
-            intent["sent"] = utt
-            intent = PadatiousIntent(**intent)
-
         intent.sent = utt
         return intent
     except Exception as e:
diff --git a/test/unittests/skills/test_utterance_intents.py b/test/unittests/skills/test_utterance_intents.py
index 431e38c7c9c1..e7bb3cb7d1e3 100644
--- a/test/unittests/skills/test_utterance_intents.py
+++ b/test/unittests/skills/test_utterance_intents.py
@@ -1,18 +1,22 @@
 import unittest
-from mycroft.skills.intent_services.padatious_service import PadatiousService, FallbackIntentContainer
-from ovos_bus_client.message import Message
+
 from ovos_utils.messagebus import FakeBus
 
+from ovos_bus_client.message import Message
+from ovos_core.intent_services.padacioso_service import FallbackIntentContainer, PadaciosoService
+
 
 class UtteranceIntentMatchingTest(unittest.TestCase):
     def get_service(self, regex_only=False, fuzz=True):
-        intent_service = PadatiousService(FakeBus(),
-                                          {"regex_only": regex_only,
-                                           "intent_cache": "~/.local/share/mycroft/intent_cache",
-                                           "train_delay": 1,
-                                           "fuzz": fuzz,
-                                           "single_thread": True,
-                                           })
+        if regex_only:
+            intent_service = PadaciosoService(FakeBus(), {"fuzz": fuzz})
+        else:
+            from ovos_core.intent_services.padatious_service import PadatiousService
+            intent_service = PadatiousService(FakeBus(),
+                                              {"intent_cache": "~/.local/share/mycroft/intent_cache",
+                                               "train_delay": 1,
+                                               "single_thread": True,
+                                               })
         # register test intents
         filename = "/tmp/test.intent"
         with open(filename, "w") as f:
@@ -24,25 +28,21 @@ def get_service(self, regex_only=False, fuzz=True):
         intent_service.register_intent(Message("padatious:register_intent", data))
         data = {'file_name': rxfilename, 'lang': 'en-US', 'name': 'test2'}
         intent_service.register_intent(Message("padatious:register_intent", data))
-        intent_service.train()
+        if not regex_only:
+            intent_service.train()
 
         return intent_service
 
     def test_padatious_intent(self):
+        try:
+            from ovos_core.intent_services.padatious_service import PadatiousService
+        except ImportError:
+            return  # skip test, padatious not installed
         intent_service = self.get_service()
 
         # assert padatious is loaded not padacioso
-        try:
-            import padatious
-            self.assertFalse(intent_service.is_regex_only)
-            self.assertFalse(intent_service.threaded_inference)
-            for container in intent_service.containers.values():
-                self.assertNotIsInstance(container, FallbackIntentContainer)
-        except ImportError:
-            self.assertTrue(intent_service.is_regex_only)
-            self.assertFalse(intent_service.threaded_inference)
-            for container in intent_service.containers.values():
-                self.assertIsInstance(container, FallbackIntentContainer)
+        for container in intent_service.containers.values():
+            self.assertNotIsInstance(container, FallbackIntentContainer)
 
         # exact match
         intent = intent_service.calc_intent("this is a test", "en-US")
@@ -67,42 +67,37 @@ def test_padatious_intent(self):
         # padaos (exact matches only) -> keep case
         # padacioso -> keep case
         # padatious -> lower case
-        if intent_service.is_regex_only:
-            self.assertEqual(intent.matches, {'thing': 'Mycroft'})
-            self.assertEqual(intent.sent, utterance)
-        else:
-            self.assertEqual(intent.matches, {'thing': 'mycroft'})
-            self.assertEqual(intent.sent, utterance)
+        self.assertEqual(intent.matches, {'thing': 'mycroft'})
+        self.assertEqual(intent.sent, utterance)
         self.assertTrue(intent.conf <= 0.9)
 
-    def test_regex_intent(self):
+    def test_padacioso_intent(self):
         intent_service = self.get_service(regex_only=True, fuzz=False)
 
-        # assert padacioso is loaded not padatious
-        self.assertTrue(intent_service.is_regex_only)
         for container in intent_service.containers.values():
-            self.assertTrue(isinstance(container, FallbackIntentContainer))
+            self.assertIsInstance(container, FallbackIntentContainer)
 
         # exact match
         intent = intent_service.calc_intent("this is a test", "en-US")
         self.assertEqual(intent.name, "test")
 
+        # fuzzy match - failure case
+        intent = intent_service.calc_intent("this test", "en-US")
+        self.assertEqual(intent.conf, 0.0)
+        self.assertTrue(intent.name is None)
+
         # regex match
-        utterance = "tell me about Mycroft"
-        intent = intent_service.calc_intent(utterance, "en-US")
+        intent = intent_service.calc_intent("tell me about Mycroft", "en-US")
         self.assertEqual(intent.name, "test2")
         self.assertEqual(intent.matches, {'thing': 'Mycroft'})
 
-        # fuzzy match - failure case (no fuzz)
-        intent = intent_service.calc_intent("this is test", "en-US")
-        self.assertTrue(intent.name is None)
-
-        # fuzzy regex match - failure case (no fuzz)
-        intent = intent_service.calc_intent("tell me everything about Mycroft",
-                                            "en-US")
+        # fuzzy regex match - failure case
+        utterance = "tell me everything about Mycroft"
+        intent = intent_service.calc_intent(utterance, "en-US")
+        self.assertEqual(intent.conf, 0.0)
         self.assertTrue(intent.name is None)
 
-    def test_regex_fuzz_intent(self):
+    def test_padacioso_fuzz_intent(self):
         intent_service = self.get_service(regex_only=True, fuzz=True)
 
         # fuzzy match - success

From 2623ef22dcf00fb060998984c912dcef3f13a550 Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Mon, 7 Aug 2023 20:08:15 +0100
Subject: [PATCH 2/5] missing skill ids in message.context

---
 ovos_core/intent_services/__init__.py         | 1 +
 ovos_core/intent_services/converse_service.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/ovos_core/intent_services/__init__.py b/ovos_core/intent_services/__init__.py
index fdad982571fb..ddba64039496 100644
--- a/ovos_core/intent_services/__init__.py
+++ b/ovos_core/intent_services/__init__.py
@@ -311,6 +311,7 @@ def handle_utterance(self, message):
 
                 if match.skill_id:
                     self.converse.activate_skill(match.skill_id)
+                    message.context["skill_id"] = match.skill_id
                     # If the service didn't report back the skill_id it
                     # takes on the responsibility of making the skill "active"
 
diff --git a/ovos_core/intent_services/converse_service.py b/ovos_core/intent_services/converse_service.py
index bdac97671962..26f1ba6eef2c 100644
--- a/ovos_core/intent_services/converse_service.py
+++ b/ovos_core/intent_services/converse_service.py
@@ -48,6 +48,7 @@ def deactivate_skill(self, skill_id, source_skill=None):
                 self.active_skills.pop(idx)
                 self.bus.emit(
                     Message("intent.service.skills.deactivated",
+                            {"skill_id": skill_id},
                             {"skill_id": skill_id}))
                 if skill_id in self._consecutive_activations:
                     self._consecutive_activations[skill_id] = 0
@@ -75,6 +76,7 @@ def activate_skill(self, skill_id, source_skill=None):
             self.active_skills.insert(0, [skill_id, time.time()])
             self.bus.emit(
                 Message("intent.service.skills.activated",
+                        {"skill_id": skill_id},
                         {"skill_id": skill_id}))
 
             self._consecutive_activations[skill_id] += 1

From ee6d168d289bc164b76f522ba168e8886ef9482d Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Mon, 7 Aug 2023 20:36:34 +0100
Subject: [PATCH 3/5] hunt down bad version

bump deps

bump deps
---
 requirements/mycroft.txt      | 4 ++--
 requirements/requirements.txt | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/requirements/mycroft.txt b/requirements/mycroft.txt
index b0f350e972b9..72729bbef5a0 100644
--- a/requirements/mycroft.txt
+++ b/requirements/mycroft.txt
@@ -4,7 +4,7 @@ mycroft-messagebus-client
 ovos_PHAL<0.1.0, >=0.0.5a9
 ovos-audio~=0.0, >=0.0.2a14
 ovos-listener~=0.0, >=0.0.2a13
-ovos-gui~=0.0, >=0.0.3a8
+ovos-gui~=0.0, >=0.0.3a11
 ovos-messagebus~=0.0, >=0.0.4a1
 
 # default plugins
@@ -16,7 +16,7 @@ ovos-ww-plugin-pocketsphinx~=0.1
 ovos-ww-plugin-precise~=0.1
 ovos-vad-plugin-webrtcvad>=0.0.1, <0.1.0
 
-ovos_plugin_common_play>=0.0.6a3, <0.1.0
+ovos_plugin_common_play>=0.0.6a4, <0.1.0
 # ovos-ocp-youtube-plugin~=0.0, >=0.0.1
 ovos-ocp-m3u-plugin>=0.0.1,<0.1.0
 ovos-ocp-rss-plugin>=0.0.2,<0.1.0
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 2ff706e83072..858a4bc4e1f6 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -6,13 +6,13 @@ combo-lock>=0.2.2, <0.3
 padacioso~=0.2, >=0.2.1a8
 adapt-parser>=1.0.0, <2.0.0
 
-ovos_bus_client<0.1.0, >=0.0.5
+ovos_bus_client<0.0.6a1, >=0.0.5
 ovos-utils<0.1.0, >=0.0.35a7
 ovos-plugin-manager<0.1.0, >=0.0.24a5
-ovos-config~=0.0,>=0.0.11a3
+ovos-config~=0.0,>=0.0.11a7
 ovos-lingua-franca>=0.4.7
 ovos_backend_client>=0.1.0a6
-ovos_workshop<0.1.0, >=0.0.12a39
+ovos_workshop<0.1.0, >=0.0.12a50
 
 # provides plugins and classic machine learning framework
 ovos-classifiers<0.1.0, >=0.0.0a33

From 2bf8596828b0c9ba733067de3ef20fbc4b05a707 Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Mon, 7 Aug 2023 21:02:42 +0100
Subject: [PATCH 4/5] pr review

---
 ovos_core/intent_services/__init__.py         | 5 +++++
 ovos_core/intent_services/converse_service.py | 8 ++++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/ovos_core/intent_services/__init__.py b/ovos_core/intent_services/__init__.py
index ddba64039496..f8afb4ec8af2 100644
--- a/ovos_core/intent_services/__init__.py
+++ b/ovos_core/intent_services/__init__.py
@@ -223,6 +223,11 @@ def disambiguate_lang(message):
         return default_lang
 
     def get_pipeline(self, skip_converse=False, skip_fallback=False):
+        """return a list of matcher functions ordered by priority
+        utterances will be sent to each matcher in order until one can handle the utterance
+        the list can be configured in mycroft.conf under intents.pipeline,
+        in the future plugins will be supported for users to define their own pipeline"""
+
         # Create matchers
         # TODO - from plugins
         if self.padatious_service is None:
diff --git a/ovos_core/intent_services/converse_service.py b/ovos_core/intent_services/converse_service.py
index 26f1ba6eef2c..0eb26934ad3b 100644
--- a/ovos_core/intent_services/converse_service.py
+++ b/ovos_core/intent_services/converse_service.py
@@ -48,8 +48,8 @@ def deactivate_skill(self, skill_id, source_skill=None):
                 self.active_skills.pop(idx)
                 self.bus.emit(
                     Message("intent.service.skills.deactivated",
-                            {"skill_id": skill_id},
-                            {"skill_id": skill_id}))
+                            data={"skill_id": skill_id},
+                            context={"skill_id": skill_id}))
                 if skill_id in self._consecutive_activations:
                     self._consecutive_activations[skill_id] = 0
 
@@ -76,8 +76,8 @@ def activate_skill(self, skill_id, source_skill=None):
             self.active_skills.insert(0, [skill_id, time.time()])
             self.bus.emit(
                 Message("intent.service.skills.activated",
-                        {"skill_id": skill_id},
-                        {"skill_id": skill_id}))
+                        data={"skill_id": skill_id},
+                        context={"skill_id": skill_id}))
 
             self._consecutive_activations[skill_id] += 1
 

From 73613e55db956a0096c7511cde79136a325d5110 Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Mon, 7 Aug 2023 21:05:18 +0100
Subject: [PATCH 5/5] list for skips instead of hardcoded fallback/converse
 kwargs

---
 ovos_core/intent_services/__init__.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/ovos_core/intent_services/__init__.py b/ovos_core/intent_services/__init__.py
index f8afb4ec8af2..e78a1212828a 100644
--- a/ovos_core/intent_services/__init__.py
+++ b/ovos_core/intent_services/__init__.py
@@ -222,7 +222,7 @@ def disambiguate_lang(message):
 
         return default_lang
 
-    def get_pipeline(self, skip_converse=False, skip_fallback=False):
+    def get_pipeline(self, skips=None):
         """return a list of matcher functions ordered by priority
         utterances will be sent to each matcher in order until one can handle the utterance
         the list can be configured in mycroft.conf under intents.pipeline,
@@ -252,11 +252,8 @@ def get_pipeline(self, skip_converse=False, skip_fallback=False):
             "padacioso_low": self.padacioso_service.match_low,
             "fallback_low": self.fallback.low_prio
         }
-        pipeline = list(self.pipeline)
-        if skip_converse and "converse" in pipeline:
-            pipeline.remove("converse")
-        if skip_fallback:
-            pipeline = [p for p in pipeline if not p.startswith("fallback_")]
+        skips = skips or []
+        pipeline = [k for k in self.pipeline if k not in skips]
         return [matchers[k] for k in pipeline]
 
     def handle_utterance(self, message):
@@ -441,7 +438,10 @@ def handle_get_intent(self, message):
         lang = get_message_lang(message)
 
         # Loop through the matching functions until a match is found.
-        for match_func in self.get_pipeline(skip_converse=True, skip_fallback=True):
+        for match_func in self.get_pipeline(skips=["converse",
+                                                   "fallback_high",
+                                                   "fallback_medium",
+                                                   "fallback_low"]):
             match = match_func([utterance], lang, message)
             if match:
                 if match.intent_type: