diff --git a/kb/use_problem_solving_method_module/models/keras/model.keras b/kb/use_problem_solving_method_module/models/keras/model.keras new file mode 100644 index 00000000..34e13db9 Binary files /dev/null and b/kb/use_problem_solving_method_module/models/keras/model.keras differ diff --git a/kb/use_problem_solving_method_module/probability.gwf b/kb/use_problem_solving_method_module/probability.gwf new file mode 100644 index 00000000..866d0650 --- /dev/null +++ b/kb/use_problem_solving_method_module/probability.gwf @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/kb/use_problem_solving_method_module/startup_test.gwf b/kb/use_problem_solving_method_module/startup_test.gwf new file mode 100644 index 00000000..90776bea --- /dev/null +++ b/kb/use_problem_solving_method_module/startup_test.gwf @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/kb/use_problem_solving_method_module/test_images/image.png b/kb/use_problem_solving_method_module/test_images/image.png new file mode 100644 index 00000000..8494f84b Binary files /dev/null and b/kb/use_problem_solving_method_module/test_images/image.png differ diff --git a/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/ConverterUser.py b/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/ConverterUser.py new file mode 100644 index 00000000..a9f80bee --- /dev/null +++ b/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/ConverterUser.py @@ -0,0 +1,15 @@ +from typing import List + +from modules.UseProblemSolvingMethodModule.converters import Converter + + +class ConverterUser: + def __init__(self, problems:List[str]): + self.problems = problems.sort() + + def __findConverter(self)->Converter: + for problem in self.problems: + match problem: + case "concept_image_classification": + pass + diff --git a/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/InterpretationParametersReader.py b/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/InterpretationParametersReader.py new file mode 100644 index 00000000..a3aefaa0 --- /dev/null +++ b/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/InterpretationParametersReader.py @@ -0,0 +1,132 @@ +import logging +from os.path import dirname, abspath +from typing import List + +import PIL.Image +import numpy + +import numpy as np +import sc_kpm +from sc_client.client import * +from sc_client.constants import sc_types +from sc_client.models import * +import tensorflow as tf + +from modules.UseProblemSolvingMethodModule.dataClasses.InterpretationParameters import InterpretationParameters +from modules.UseProblemSolvingMethodModule.dataClasses.AnnStruct import AnnStruct +from modules.UseProblemSolvingMethodModule.dataClasses.inputOutput.ImageStruct import Image + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s | %(name)s | %(message)s", datefmt="[%d-%b-%y %H:%M:%S]" +) + +# todo: use absolute filepaths? +def get_ann_path() -> str: + script_directory = dirname(dirname(abspath(__file__))) + ann_path = dirname(dirname(dirname(dirname(dirname(script_directory))))) + return ann_path + + +class InterpretationParametersReader: + def __init__(self) -> None: + pass + + def get_interpretation_parameters(self, action_addr: ScAddr) -> InterpretationParameters: + ann_address = self.__get_ann_address(action_addr) + + ann_model = self.__get_keras_model(ann_address) + ann_input_shape = self.__get_ann_input_shape(ann_address) + ann_output_shape = self.__get_ann_output_shape(ann_address) + + ann_struct = AnnStruct(network_address=ann_address, + ann_model=ann_model, + num_of_inputs=ann_input_shape, + num_of_outputs=ann_output_shape) + + problem_addr = self.__get_problem_addr(action_addr) + input_struct = self.__get_image(self.__get_image_addr(problem_addr)) + + return InterpretationParameters(ann_struct, input_struct) + + @staticmethod + def __get_rrel_target(source_addr: ScAddr, rrel_name: str) -> ScAddr: + template = ScTemplate() + template.triple_with_relation( + source_addr, + sc_types.EDGE_ACCESS_VAR_POS_PERM, + sc_types.NODE_VAR, + sc_types.EDGE_ACCESS_VAR_POS_PERM, + sc_kpm.ScKeynodes[rrel_name] + ) + return template_search(template)[0][2] + + @staticmethod + def __get_nrel_target_link(source_addr: ScAddr, nrel_name: str) -> ScAddr: + template = ScTemplate() + template.triple_with_relation( + source_addr, + sc_types.EDGE_D_COMMON_VAR, + sc_types.LINK_VAR, + sc_types.EDGE_ACCESS_VAR_POS_PERM, + sc_kpm.ScKeynodes[nrel_name] + ) + return template_search(template)[0][2] + + # def __get_classes(source_addr: ScAddr) -> List[ScAddr]: + # template = ScTemplate() + # template.triple( + # sc_types.NODE_CLASS, + # sc_types.EDGE_ACCESS_VAR_POS_PERM, + # source_addr + # ) + # + # sc_classes = [] + # for template_result in template_search(template): + # sc_classes.append(template_result[2]) + # return sc_classes + + def __get_problem_addr(self, action_addr: ScAddr) -> ScAddr: + return self.__get_rrel_target(action_addr, "rrel_1") + + # def __get_problem_types(self, problem_addr: ScAddr) -> List[ScAddr]: + # return self.__get_classes(problem_addr) + + def __get_image_addr(self, problem_addr: ScAddr) -> ScAddr: + return self.__get_rrel_target(problem_addr, "rrel_input_image") + + def __get_file(self, file_addr: ScAddr, nrel_name: str) -> str: + filepath_link_addr = self.__get_nrel_target_link(file_addr, nrel_name) + file_filepath = get_link_content(filepath_link_addr)[0].data + return file_filepath + + def __get_image(self, image_addr: ScAddr) -> Image: + image_path = self.__get_file(image_addr, "nrel_filepath") + kb_path = f'{get_ann_path()}/kb' + + image = PIL.Image.open(f'{kb_path}/{image_path}') + image_struct = Image(object_addr=image_addr, + data_array=numpy.array(image), + image_shape=image.size) + logging.info(f'Image imported') + return image_struct + + def __get_ann_address(self, action_addr: ScAddr) -> ScAddr: + return self.__get_rrel_target(action_addr, "rrel_2") + + def __get_keras_model(self, model_addr: ScAddr) -> tf.keras.Model: + ann_path = self.__get_file(model_addr, "nrel_keras_ann") + kb_path = f'{get_ann_path()}/kb' + + model = tf.keras.models.load_model(f"{kb_path}/{ann_path}") + return model + + def __get_ann_input_shape(self, ann_addr: ScAddr) -> int: + model = self.__get_keras_model(ann_addr) + logging.info(f"Input shape: {model.input_shape[1]}") + return model.input_shape[1] + + def __get_ann_output_shape(self, ann_addr: ScAddr) -> int: + model = self.__get_keras_model(ann_addr) + logging.info(f"Output shape: {model.output_shape[1]}") + return model.output_shape[1] + diff --git a/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/UseProblemSolvingMethodAgent.py b/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/UseProblemSolvingMethodAgent.py new file mode 100644 index 00000000..81dc9b43 --- /dev/null +++ b/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/UseProblemSolvingMethodAgent.py @@ -0,0 +1,101 @@ +import sc_kpm +from sc_client.client import create_elements, template_search, resolve_keynodes +from sc_client.constants import sc_types +from sc_client.constants.common import ScEventType +from sc_client.models import ScAddr, ScTemplate, ScConstruction, ScIdtfResolveParams +from sc_kpm import ScAgentClassic, ScKeynodes +from sc_kpm.sc_result import ScResult +from sc_kpm.utils import get_system_idtf, create_node, create_edge, get_link_content_data +import tensorflow as tf +import numpy + +from modules.UseProblemSolvingMethodModule.annInterpreter.InterpretationParametersReader import \ + InterpretationParametersReader +from modules.UseProblemSolvingMethodModule.converters.ImageConverter import ImageConverter + +from modules.UseProblemSolvingMethodModule.dataClasses.InterpretationParameters import InterpretationParameters + + + +class UseProblemSolvingMethodAgent(ScAgentClassic): + def __init__(self) -> None: + super().__init__("action_use_problem_solving_method") + + def on_event(self, event_element: ScAddr, event_edge: ScAddr, action_element: ScAddr) -> ScResult: + self.logger.info("UseProblemSolvingMethodAgent started") + result = self.__run(action_element) + self.logger.info("UseProblemSolvingAgent finished") + return result + + def __run(self, action_element: ScAddr) -> ScResult: + # Get Interpretation parameters from input struct + reader = InterpretationParametersReader() + interpretation_parameters: InterpretationParameters = reader.get_interpretation_parameters(action_element) + + model = interpretation_parameters.ann_struct.ann_model + input_data = ImageConverter.convert_image_to_inputs(interpretation_parameters.input_struct, + interpretation_parameters.ann_struct.num_of_inputs) + + predictions = model.predict(input_data) + score = tf.nn.softmax(predictions[0]) + print(numpy.max(score * 100)) + self.logger.info(f"Object score:{score}") + + self.__save_ann_output(predictions, interpretation_parameters) + + output_construct = ScConstruction() + output_construct.create_edge(sc_type=sc_types.EDGE_D_COMMON_CONST, + src=action_element, + trg=interpretation_parameters.input_struct.object_addr, + alias='edge') + output_construct.create_edge(sc_types.EDGE_ACCESS_CONST_POS_PERM, + sc_kpm.ScKeynodes["nrel_result"], + 'edge') + addresses = create_elements(output_construct) + assert all(addresses) + + return ScResult.OK + + def __save_ann_output(self, predictions: numpy.array, interpretation_parameters: InterpretationParameters) -> None: + objet_addr = interpretation_parameters.input_struct.object_addr + + score = tf.nn.softmax(predictions[0]) + max_prediction: int = numpy.max(score * 100) + max_prediction_index: int = int(numpy.argmax(score * 100)) + self.__save_ann_output_to_sc_memory(max_prediction, max_prediction_index, objet_addr) + + def __save_ann_output_to_sc_memory(self, max_prediction: int, max_prediction_index: int, + object_addr: ScAddr) -> None: + self.logger.info(f"Max prediction: {max_prediction}") + + # TODO add ann classes if exists + # Now it name recognised class as index of output array + result_node_params = ScIdtfResolveParams(idtf=f'[{max_prediction_index}]', type=sc_types.NODE_CONST_CLASS) + result_node_addr = resolve_keynodes(result_node_params)[0] + construction = ScConstruction() + construction.create_edge(sc_type=sc_types.EDGE_ACCESS_CONST_FUZ_PERM, + src=result_node_addr, + trg=object_addr, + alias='fuz_edge') + addresses = create_elements(construction) + assert len(addresses) == 1 + assert all(addresses) + fuz_edge_addr = addresses[0] + self.logger.info(f'Fuz Edge addr:{fuz_edge_addr}') + + probability_node_params = ScIdtfResolveParams(idtf=f'{str(max_prediction)}%', type=sc_types.NODE_CONST_CLASS) + probability_node_addr = resolve_keynodes(probability_node_params)[0] + probability_construction = ScConstruction() + probability_construction.create_edge(sc_type=sc_types.EDGE_ACCESS_CONST_POS_PERM, + src=sc_kpm.ScKeynodes["probability"], + trg=probability_node_addr, ) + probability_construction.create_edge(sc_type=sc_types.EDGE_ACCESS_CONST_POS_PERM, + src=probability_node_addr, + trg=fuz_edge_addr) + addresses = create_elements(probability_construction) + assert len(addresses) == 2 + assert all(addresses) + # fuz_edge_addr = addresses[0] + self.logger.info(f'Probability created') + + self.logger.info(f"Saved to kb") diff --git a/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/UseProblemSolvingMethodModule.py b/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/UseProblemSolvingMethodModule.py new file mode 100644 index 00000000..5f9c6b0e --- /dev/null +++ b/problem-solver/py/modules/UseProblemSolvingMethodModule/annInterpreter/UseProblemSolvingMethodModule.py @@ -0,0 +1,8 @@ +from sc_kpm import ScModule + +from .UseProblemSolvingMethodAgent import UseProblemSolvingMethodAgent + + +class UseProblemSolvingMethodModule(ScModule): + def __init__(self) -> None: + super().__init__(UseProblemSolvingMethodAgent()) \ No newline at end of file diff --git a/problem-solver/py/modules/UseProblemSolvingMethodModule/converters/Converter.py b/problem-solver/py/modules/UseProblemSolvingMethodModule/converters/Converter.py new file mode 100644 index 00000000..5090e1bd --- /dev/null +++ b/problem-solver/py/modules/UseProblemSolvingMethodModule/converters/Converter.py @@ -0,0 +1,3 @@ +class Converter: + def __init__(self): + pass \ No newline at end of file diff --git a/problem-solver/py/modules/UseProblemSolvingMethodModule/converters/ImageConverter.py b/problem-solver/py/modules/UseProblemSolvingMethodModule/converters/ImageConverter.py new file mode 100644 index 00000000..d17db866 --- /dev/null +++ b/problem-solver/py/modules/UseProblemSolvingMethodModule/converters/ImageConverter.py @@ -0,0 +1,14 @@ +import numpy + +from .Converter import Converter +from modules.UseProblemSolvingMethodModule.dataClasses.inputOutput.ImageStruct import Image + + +class ImageConverter(Converter): + @staticmethod + def convert_image_to_inputs(image: Image, number_of_inputs: int) -> numpy.array: + numpydata = image.data_array + numpy.shape(numpydata) + + numpydata = numpy.reshape(numpydata, (1, number_of_inputs)) + return numpydata diff --git a/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/AnnStruct.py b/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/AnnStruct.py new file mode 100644 index 00000000..99122249 --- /dev/null +++ b/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/AnnStruct.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from typing import List + +import keras +from sc_client.models import ScAddr + + +@dataclass +class AnnStruct: + network_address: ScAddr + ann_model: keras.Model + num_of_inputs: int + num_of_outputs: int + output_labels: List[str] = None \ No newline at end of file diff --git a/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/InterpretationParameters.py b/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/InterpretationParameters.py new file mode 100644 index 00000000..83ec96ac --- /dev/null +++ b/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/InterpretationParameters.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass +from typing import List + +from sc_client.models import ScAddr + +from .AnnStruct import AnnStruct +from .inputOutput.ImageStruct import Image + + +@dataclass +class InterpretationParameters: + ann_struct: AnnStruct + input_struct: Image # inputs for ann + problem_types: List[ScAddr] = None # todo or maybe unused + + diff --git a/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/inputOutput/ImageStruct.py b/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/inputOutput/ImageStruct.py new file mode 100644 index 00000000..4eec9286 --- /dev/null +++ b/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/inputOutput/ImageStruct.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from .InputOutputStruct import InputOutput + +import numpy as np + + +@dataclass +class Image(InputOutput): + # data_array: np.array + image_shape: tuple = None + label: str = None \ No newline at end of file diff --git a/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/inputOutput/InputOutputStruct.py b/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/inputOutput/InputOutputStruct.py new file mode 100644 index 00000000..dddf97df --- /dev/null +++ b/problem-solver/py/modules/UseProblemSolvingMethodModule/dataClasses/inputOutput/InputOutputStruct.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + +import numpy as np +from sc_client.models import ScAddr + + +@dataclass +class InputOutput: + object_addr: ScAddr + data_array: np.array diff --git a/problem-solver/py/requirements.txt b/problem-solver/py/requirements.txt index 4a220a1a..bd31d3a4 100644 --- a/problem-solver/py/requirements.txt +++ b/problem-solver/py/requirements.txt @@ -1,5 +1,7 @@ py-sc-kpm==0.2.0 py-sc-client==0.3.0 requests==2.31.0 -numpy +numpy~=1.26.4 tensorflow==2.16.1 +pandas~=2.2.2 +pillow~=11.0.0 \ No newline at end of file diff --git a/problem-solver/py/server.py b/problem-solver/py/server.py index dc3a59d3..429b83b8 100644 --- a/problem-solver/py/server.py +++ b/problem-solver/py/server.py @@ -1,5 +1,7 @@ import argparse from sc_kpm import ScServer + +from modules.UseProblemSolvingMethodModule.annInterpreter.UseProblemSolvingMethodModule import UseProblemSolvingMethodModule from modules.messageProcessingModule.FnnAgentProcessingModule import FnnAgentProcessingModule from modules.messageProcessingModule.FnnTrainerProcessingModule import FnnTrainerProcessingModule from pathlib import Path @@ -20,7 +22,8 @@ def main(args: dict): with server.connect(): modules = [ FnnAgentProcessingModule(), - FnnTrainerProcessingModule() + FnnTrainerProcessingModule(), + UseProblemSolvingMethodModule() ] server.add_modules(*modules) with server.register_modules(): diff --git a/problem-solver/py/tests/interpretation_testcase.py b/problem-solver/py/tests/interpretation_testcase.py new file mode 100644 index 00000000..e69de29b