diff --git a/birds_nest/pybirdai/entry_points/convert_ldm_to_sdd_hierarchies.py b/birds_nest/pybirdai/entry_points/convert_ldm_to_sdd_hierarchies.py new file mode 100644 index 000000000..a2043a7ea --- /dev/null +++ b/birds_nest/pybirdai/entry_points/convert_ldm_to_sdd_hierarchies.py @@ -0,0 +1,49 @@ +# coding=UTF-8 +# Copyright (c) 2024 Bird Software Solutions Ltd +# This program and the accompanying materials +# are made available under the terms of the Eclipse Public License 2.0 +# which accompanies this distribution, and is available at +# https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Neil Mackenzie - initial API and implementation +# + +import os +from django.apps import AppConfig +from pybirdai.context.sdd_context_django import SDDContext +from django.conf import settings + +class RunConvertLDMToSDDHierarchies(AppConfig): + """ + Django AppConfig for converting LDM hierarchies to SDD hierarchies. + """ + + path = os.path.join(settings.BASE_DIR, 'birds_nest') + + @staticmethod + def run_convert_hierarchies(): + """ + Execute the process of converting LDM hierarchies to SDD hierarchies. + """ + from pybirdai.process_steps.hierarchy_conversion.convert_ldm_to_sdd_hierarchies import ( + ConvertLDMToSDDHierarchies + ) + from pybirdai.context.context import Context + + base_dir = settings.BASE_DIR + sdd_context = SDDContext() + sdd_context.file_directory = os.path.join(base_dir, 'resources') + sdd_context.output_directory = os.path.join(base_dir, 'results') + + context = Context() + context.file_directory = sdd_context.file_directory + context.output_directory = sdd_context.output_directory + + ConvertLDMToSDDHierarchies().convert_hierarchies(context, sdd_context) + + def ready(self): + # This method is still needed for Django's AppConfig + pass \ No newline at end of file diff --git a/birds_nest/pybirdai/process_steps/hierarchy_conversion/convert_ldm_to_sdd_hierarchies.py b/birds_nest/pybirdai/process_steps/hierarchy_conversion/convert_ldm_to_sdd_hierarchies.py new file mode 100644 index 000000000..3af9f3ada --- /dev/null +++ b/birds_nest/pybirdai/process_steps/hierarchy_conversion/convert_ldm_to_sdd_hierarchies.py @@ -0,0 +1,237 @@ +# coding=UTF-8 +# Copyright (c) 2024 Bird Software Solutions Ltd +# This program and the accompanying materials +# are made available under the terms of the Eclipse Public License 2.0 +# which accompanies this distribution, and is available at +# https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Neil Mackenzie - initial API and implementation +# + +import os +import csv +from datetime import datetime +from django.apps import apps +from django.db import models +from difflib import get_close_matches + +class ConvertLDMToSDDHierarchies: + """Class for converting LDM hierarchies to SDD hierarchies.""" + + def find_closest_member(self, member_id): + """ + Find the closest matching existing member name. + + Args: + member_id: The member ID to find matches for + + Returns: + tuple: (name, member_id) of closest match, or (None, None) if no matches + """ + MEMBER = apps.get_model('pybirdai', 'MEMBER') + + # Check for exact match with underscores + member_with_underscores = member_id.replace(' ', '_') + member = MEMBER.objects.filter(name=member_with_underscores).first() + if member: + return (member.name, member.member_id) + + # If no underscore match, find closest match + existing_members = list(MEMBER.objects.values_list('name', flat=True)) + matches = get_close_matches(member_id, existing_members, n=1, cutoff=0.6) + if matches: + member = MEMBER.objects.filter(name=matches[0]).first() + return (member.name, member.member_id) + return (None, None) + + def check_member_exists(self, member_id): + """ + Check if a member exists in the MEMBER table. + + Args: + member_id: The member ID to check (with spaces) + + Returns: + tuple: (exists, member_id) where exists is bool and member_id is the matched ID or None + """ + MEMBER = apps.get_model('pybirdai', 'MEMBER') + # Check with spaces + member = MEMBER.objects.filter(name=member_id).first() + if member: + return (True, member.member_id) + # Check with underscores + member = MEMBER.objects.filter(name=member_id.replace(' ', '_')).first() + if member: + return (True, member.member_id) + return (False, None) + + def get_all_subclasses_and_delegates(self, cls, processed=None, parent=None, level=1): + """ + Recursively get all subclasses and delegate relationships of a class. + + Args: + cls: The class to get subclasses and delegates for + processed: Set of already processed classes to avoid cycles + parent: Parent class for tracking hierarchy + level: Current level in hierarchy + + Returns: + list: List of tuples (class, parent_class, is_delegate, level) + """ + if processed is None: + processed = set() + + if cls in processed: + return [] + + processed.add(cls) + result = [] + + # Get direct subclasses + for subclass in cls.__subclasses__(): + result.append((subclass, cls, False, level)) + result.extend(self.get_all_subclasses_and_delegates(subclass, processed, cls, level + 1)) + + # Get delegate relationships + for field in cls._meta.get_fields(): + if isinstance(field, models.ForeignKey) and field.name.endswith('_delegate'): + delegate_class = field.related_model + if delegate_class not in processed: + result.append((delegate_class, cls, True, level)) + result.extend(self.get_all_subclasses_and_delegates(delegate_class, processed, cls, level + 1)) + + return result + + def convert_hierarchies(self, context, sdd_context): + """ + Convert LDM hierarchies to SDD hierarchies. + + Args: + context: The general context containing file paths and settings + sdd_context: The SDD-specific context containing SDD-related settings + """ + # Constants for the hierarchy + MAINTENANCE_AGENCY_ID = "BIRD" + HIERARCHY_ID = "INSTRMNT_HIERARCHY" + DOMAIN_ID = "INSTRMNT_DOMAIN" + VALID_FROM = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + VALID_TO = "9999-12-31" + + # Get the INSTRMNT class + INSTRMNT = apps.get_model('pybirdai', 'INSTRMNT') + + # Get all subclasses and delegates recursively + class_relationships = self.get_all_subclasses_and_delegates(INSTRMNT) + + # Track missing members and their suggestions + missing_members = {} + + # Create output directory if it doesn't exist + output_dir = os.path.join(context.output_directory, 'ldm_to_sdd_hierarchies') + os.makedirs(output_dir, exist_ok=True) + + # Create member_hierarchy.csv + hierarchy_file = os.path.join(output_dir, 'member_hierarchy.csv') + with open(hierarchy_file, 'w', newline='') as f: + writer = csv.writer(f) + # Write header + writer.writerow([ + 'MAINTENANCE_AGENCY_ID', + 'MEMBER_HIERARCHY_ID', + 'CODE', + 'DOMAIN_ID', + 'NAME', + 'DESCRIPTION', + 'IS_MAIN_HIERARCHY' + ]) + # Write data + writer.writerow([ + MAINTENANCE_AGENCY_ID, + HIERARCHY_ID, + '1', + DOMAIN_ID, + 'Instrument type hierarchy', + 'Hierarchical structure of instrument types and delegates', + 'true' + ]) + + # Create member_hierarchy_node.csv + nodes_file = os.path.join(output_dir, 'member_hierarchy_node.csv') + with open(nodes_file, 'w', newline='') as f: + writer = csv.writer(f) + # Write header + writer.writerow([ + 'MEMBER_HIERARCHY_ID', + 'MEMBER_ID', + 'LEVEL', + 'PARENT_MEMBER_ID', + 'COMPARATOR', + 'OPERATOR', + 'VALID_FROM', + 'VALID_TO' + ]) + + # Check root node + root_member_id = INSTRMNT._meta.verbose_name.replace('_', ' ') + exists, matched_id = self.check_member_exists(root_member_id) + if not exists: + suggestion = self.find_closest_member(root_member_id) + missing_members[root_member_id] = suggestion + + # Write root node + writer.writerow([ + HIERARCHY_ID, + matched_id if matched_id else root_member_id, + 1, + '', + '=', + '', + VALID_FROM, + VALID_TO + ]) + + # Write all nodes (both inheritance and delegate relationships) + for cls, parent_cls, is_delegate, level in class_relationships: + member_name = cls._meta.verbose_name.replace('_', ' ') + parent_member_name = parent_cls._meta.verbose_name.replace('_', ' ') + + # Check if members exist and get their IDs + exists, member_matched_id = self.check_member_exists(member_name) + if not exists: + suggestion = self.find_closest_member(member_name) + missing_members[member_name] = suggestion + + exists, parent_matched_id = self.check_member_exists(parent_member_name) + if not exists: + suggestion = self.find_closest_member(parent_member_name) + missing_members[parent_member_name] = suggestion + + writer.writerow([ + HIERARCHY_ID, + member_matched_id if member_matched_id else member_name, + level + 1, # Add 1 since root is level 1 + parent_matched_id if parent_matched_id else parent_member_name, + '=' if not is_delegate else 'D', # Use 'D' comparator for delegate relationships + '', + VALID_FROM, + VALID_TO + ]) + + # Save missing members information to CSV + if missing_members: + missing_members_file = os.path.join(output_dir, 'missing_members.csv') + with open(missing_members_file, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(['Missing Member', 'Match Type', 'Matched Name', 'Matched Member ID']) + for member, suggestion in sorted(missing_members.items()): + if suggestion: + name, member_id = suggestion + match_type = 'Underscore Match' if name == member.replace(' ', '_') else 'Closest Match' + writer.writerow([member, match_type, name, member_id]) + else: + writer.writerow([member, 'No Match', '', '']) + + return f"Created hierarchy files in {output_dir} including both inheritance and delegate relationships" \ No newline at end of file diff --git a/birds_nest/pybirdai/templates/pybirdai/bird_diffs_and_corrections.html b/birds_nest/pybirdai/templates/pybirdai/bird_diffs_and_corrections.html new file mode 100644 index 000000000..c13c0adf4 --- /dev/null +++ b/birds_nest/pybirdai/templates/pybirdai/bird_diffs_and_corrections.html @@ -0,0 +1,39 @@ + +{% extends 'base.html' %} +{% load static %} + +{% block title %}Create Input Structures{% endblock %} + +{% block content %} + +

BIRD Export Diffs and Corrections

+
+ + Export DB to CSV + Export Database to CSV Files + + + Convert Hierarchies + Convert LDM Hierarchies to SDD Hierarchies (Use only with LDM model loaded) + + + View Results + View LDM to SDD Hierarchy Conversion Results + + + back_arrow + Back to the PyBIRD AI Home Page + +
+{% endblock %} \ No newline at end of file diff --git a/birds_nest/pybirdai/templates/pybirdai/home.html b/birds_nest/pybirdai/templates/pybirdai/home.html index fd03e94de..955cd64b9 100644 --- a/birds_nest/pybirdai/templates/pybirdai/home.html +++ b/birds_nest/pybirdai/templates/pybirdai/home.html @@ -43,10 +43,10 @@

Home

Concept icon View Populated Templates - - - Export DB to CSV - Export Database to CSV Files + + + BIRD Diffs + BIRD Export,Diffs and Corrections diff --git a/birds_nest/pybirdai/templates/pybirdai/view_ldm_to_sdd_results.html b/birds_nest/pybirdai/templates/pybirdai/view_ldm_to_sdd_results.html new file mode 100644 index 000000000..053383f11 --- /dev/null +++ b/birds_nest/pybirdai/templates/pybirdai/view_ldm_to_sdd_results.html @@ -0,0 +1,90 @@ + +{% extends 'base.html' %} + +{% block title %}LDM to SDD Hierarchy Conversion Results{% endblock %} + +{% block content %} +

LDM to SDD Hierarchy Conversion Results

+Back to BIRD Export Diffs and Corrections + +{% if not csv_data %} +

No conversion results found. Please run the conversion first.

+{% else %} + {% for filename, data in csv_data.items %} +

{{ filename }}

+
+ + + + {% for header in data.headers %} + + {% endfor %} + + + + {% for row in data.rows %} + + {% for cell in row %} + + {% endfor %} + + {% endfor %} + +
{{ header }}
{{ cell }}
+
+ {% endfor %} +{% endif %} + + +{% endblock %} \ No newline at end of file diff --git a/birds_nest/pybirdai/urls.py b/birds_nest/pybirdai/urls.py index ecd33ddd1..d6e38fac4 100644 --- a/birds_nest/pybirdai/urls.py +++ b/birds_nest/pybirdai/urls.py @@ -106,4 +106,7 @@ path('add_member_mapping_item/', views.add_member_mapping_item, name='add_member_mapping_item'), path('view_member_mapping_items_by_row/', views.view_member_mapping_items_by_row, name='view_member_mapping_items_by_row'), path('export-database-to-csv/', views.export_database_to_csv, name='export_database_to_csv'), + path('bird_diffs_and_corrections/', views.bird_diffs_and_corrections, name='bird_diffs_and_corrections'), + path('convert_ldm_to_sdd_hierarchies/', views.convert_ldm_to_sdd_hierarchies, name='convert_ldm_to_sdd_hierarchies'), + path('view_ldm_to_sdd_results/', views.view_ldm_to_sdd_results, name='view_ldm_to_sdd_results'), ] \ No newline at end of file diff --git a/birds_nest/pybirdai/views.py b/birds_nest/pybirdai/views.py index 25d7438a8..6f82047b1 100644 --- a/birds_nest/pybirdai/views.py +++ b/birds_nest/pybirdai/views.py @@ -39,6 +39,7 @@ from .entry_points.upload_sqldev_eldm_files import UploadSQLDevELDMFiles from .entry_points.upload_technical_export_files import UploadTechnicalExportFiles from .entry_points.create_django_models import RunCreateDjangoModels +from .entry_points.convert_ldm_to_sdd_hierarchies import RunConvertLDMToSDDHierarchies import os import csv from pathlib import Path @@ -1591,3 +1592,43 @@ def export_database_to_csv(request): return response +def bird_diffs_and_corrections(request): + """ + View function for displaying BIRD diffs and corrections page. + """ + return render(request, 'pybirdai/bird_diffs_and_corrections.html') + +def convert_ldm_to_sdd_hierarchies(request): + """View for converting LDM hierarchies to SDD hierarchies.""" + if request.GET.get('execute') == 'true': + try: + RunConvertLDMToSDDHierarchies.run_convert_hierarchies() + return JsonResponse({'status': 'success'}) + except Exception as e: + return JsonResponse({'status': 'error', 'message': str(e)}) + + return create_response_with_loading( + request, + 'Converting LDM Hierarchies to SDD Hierarchies', + 'Successfully converted LDM hierarchies to SDD hierarchies.', + reverse('pybirdai:bird_diffs_and_corrections'), + 'BIRD Export Diffs and Corrections' + ) + +def view_ldm_to_sdd_results(request): + """View for displaying the LDM to SDD hierarchy conversion results.""" + results_dir = os.path.join(settings.BASE_DIR, 'results', 'ldm_to_sdd_hierarchies') + + # Read the CSV files + csv_data = {} + for filename in ['member_hierarchy.csv', 'member_hierarchy_node.csv', 'missing_members.csv']: + filepath = os.path.join(results_dir, filename) + if os.path.exists(filepath): + with open(filepath, 'r', newline='') as f: + reader = csv.reader(f) + headers = next(reader) # Get headers + rows = list(reader) # Get data rows + csv_data[filename] = {'headers': headers, 'rows': rows} + + return render(request, 'pybirdai/view_ldm_to_sdd_results.html', {'csv_data': csv_data}) + diff --git a/birds_nest/results/ldm_to_sdd_hierarchies/tmp b/birds_nest/results/ldm_to_sdd_hierarchies/tmp new file mode 100644 index 000000000..e69de29bb diff --git a/birds_nest/static/images/bird_diffs.webp b/birds_nest/static/images/bird_diffs.webp new file mode 100644 index 000000000..8a8379aa2 Binary files /dev/null and b/birds_nest/static/images/bird_diffs.webp differ