Skip to content

Commit

Permalink
remvoed code duplication in data_export classes
Browse files Browse the repository at this point in the history
  • Loading branch information
@expeditionengineer
expeditionengineer committed Dec 13, 2024
1 parent 70dde94 commit 7cd9db4
Show file tree
Hide file tree
Showing 5 changed files with 10 additions and 487 deletions.
124 changes: 2 additions & 122 deletions webcentral/src/Datasets/data_export.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@

from tools_over.models import Tools
from .models import Dataset
from common.data_export import AbstractDataExport


class DataExport:
class DataExport(AbstractDataExport):
""" """

SEPARATOR_M2M = ";;"
Expand Down Expand Up @@ -40,123 +40,3 @@ class DataExport:

DATA_APP_DIR = "02_tool_over"
EXPORT_MODEL = Dataset

def __init__(self, filename):
""" """
self.filename = filename

def exportToXlsx(self):
""" """

allObjs = self.EXPORT_MODEL.objects.all()

germanData, englishData = self._sortObjectsIntoGermanAndEnglishDs(
allObjs
)

self._writeDictsToXlsx(germanData, englishData)

def _sortObjectsIntoGermanAndEnglishDs(self, ormObjs) -> tuple:
"""Sort the the german and english attributes of the ORM-objects into to different dictionaries.
Arguments:
ormObjs: Queryset
Iterable of orm-objs to be sorted into german and english buckets
Returns:
tuple of germanData and englishData
"""
germanData = {}
englishData = {}

for ormObj in ormObjs:

for mappingNameORM in self.MAPPING_ORM_TO_XLSX.keys():
if hasattr(ormObj, mappingNameORM + "_en"):
englishData = self._checkIfKeyExistsAndAppendData(
englishData, mappingNameORM
)
englishData = self._apppendToDs(
englishData, mappingNameORM, ormObj, "_en"
)

germanData = self._checkIfKeyExistsAndAppendData(
germanData, mappingNameORM
)
germanData = self._apppendToDs(
germanData, mappingNameORM, ormObj, "_de"
)

else:
englishData = self._checkIfKeyExistsAndAppendData(
englishData, mappingNameORM
)
englishData = self._apppendToDs(
englishData, mappingNameORM, ormObj, "_en"
)

germanData = self._checkIfKeyExistsAndAppendData(
germanData, mappingNameORM
)
germanData = self._apppendToDs(
germanData, mappingNameORM, ormObj, "_de"
)

return (germanData, englishData)

def _checkIfKeyExistsAndAppendData(self, dataDict, keyName) -> dict:
"""Check if the key in the data-dict exists otherwise create it.
Arguments:
dataDict: dict
datastructure to store a table. keys are the column name.
keyName: str
name of the key to be checked if a key in the dict.
Returns:
dict
"""
if keyName in dataDict.keys():
return dataDict

dataDict[keyName] = []
return dataDict

def _apppendToDs(self, dataDict, mappingNameORM, ormObj, languageSuffix):
""" """
if isinstance(
ormObj._meta.get_field(mappingNameORM), models.ManyToManyField
):
concatenatedMTMStr = ormObj.getManyToManyAttrAsStr(
mappingNameORM, languageSuffix, separator=";;"
)
dataDict[mappingNameORM].append(concatenatedMTMStr)
elif isinstance(
ormObj._meta.get_field(mappingNameORM), models.BooleanField
):
valueOfBooleanField = getattr(ormObj, mappingNameORM)
if valueOfBooleanField == "" or valueOfBooleanField is None:
dataDict[mappingNameORM].append("")
else:
dataDict[mappingNameORM].append(
int(getattr(ormObj, mappingNameORM))
)
else:
try:
dataDict[mappingNameORM].append(
getattr(ormObj, mappingNameORM + languageSuffix)
)
except AttributeError:
dataDict[mappingNameORM].append(getattr(ormObj, mappingNameORM))

return dataDict

def _writeDictsToXlsx(self, germanData, englishData):
""" """
dfGerman = pd.DataFrame(germanData)
dfEnglish = pd.DataFrame(englishData)

with pd.ExcelWriter(self.filename, engine="openpyxl") as writer:
dfGerman.to_excel(writer, sheet_name="German", index=False)
dfEnglish.to_excel(writer, sheet_name="English", index=False)
124 changes: 2 additions & 122 deletions webcentral/src/TechnicalStandards/data_export.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
from django.db import models

from .models import Norm
from common.data_export import AbstractDataExport


class DataExport:
class DataExport(AbstractDataExport):
""" """

SEPARATOR_M2M = ";;"
Expand All @@ -20,123 +20,3 @@ class DataExport:

DATA_APP_DIR = "05_technical_standards"
EXPORT_MODEL_OBJ = Norm

def __init__(self, filename):
""" """
self.filename = filename

def exportToXlsx(self):
""" """

allProtocols = self.EXPORT_MODEL_OBJ.objects.all()

germanData, englishData = self._sortObjectsIntoGermanAndEnglishDs(
allProtocols
)

self._writeDictsToXlsx(germanData, englishData)

def _sortObjectsIntoGermanAndEnglishDs(self, ormObjs) -> tuple:
"""Sort the the german and english attributes of the ORM-objects into to different dictionaries.
Arguments:
ormObjs: Queryset
Iterable of orm-objs to be sorted into german and english buckets
Returns:
tuple of germanData and englishData
"""
germanData = {}
englishData = {}

for ormObj in ormObjs:

for mappingNameORM in self.MAPPING_ORM_TO_XLSX.keys():
if hasattr(ormObj, mappingNameORM + "_en"):
englishData = self._checkIfKeyExistsAndAppendData(
englishData, mappingNameORM
)
englishData = self._apppendToDs(
englishData, mappingNameORM, ormObj, "_en"
)

germanData = self._checkIfKeyExistsAndAppendData(
germanData, mappingNameORM
)
germanData = self._apppendToDs(
germanData, mappingNameORM, ormObj, "_de"
)

else:
englishData = self._checkIfKeyExistsAndAppendData(
englishData, mappingNameORM
)
englishData = self._apppendToDs(
englishData, mappingNameORM, ormObj, "_en"
)

germanData = self._checkIfKeyExistsAndAppendData(
germanData, mappingNameORM
)
germanData = self._apppendToDs(
germanData, mappingNameORM, ormObj, "_de"
)

return (germanData, englishData)

def _checkIfKeyExistsAndAppendData(self, dataDict, keyName) -> dict:
"""Check if the key in the data-dict exists otherwise create it.
Arguments:
dataDict: dict
datastructure to store a table. keys are the column name.
keyName: str
name of the key to be checked if a key in the dict.
Returns:
dict
"""
if keyName in dataDict.keys():
return dataDict

dataDict[keyName] = []
return dataDict

def _apppendToDs(self, dataDict, mappingNameORM, ormObj, languageSuffix):
""" """
if isinstance(
ormObj._meta.get_field(mappingNameORM), models.ManyToManyField
):
concatenatedMTMStr = ormObj.getManyToManyAttrAsStr(
mappingNameORM, languageSuffix, separator=";;"
)
dataDict[mappingNameORM].append(concatenatedMTMStr)
elif isinstance(
ormObj._meta.get_field(mappingNameORM), models.BooleanField
):
valueOfBooleanField = getattr(ormObj, mappingNameORM)
if valueOfBooleanField == "" or valueOfBooleanField is None:
dataDict[mappingNameORM].append("")
else:
dataDict[mappingNameORM].append(
int(getattr(ormObj, mappingNameORM))
)
else:
try:
dataDict[mappingNameORM].append(
getattr(ormObj, mappingNameORM + languageSuffix)
)
except AttributeError:
dataDict[mappingNameORM].append(getattr(ormObj, mappingNameORM))

return dataDict

def _writeDictsToXlsx(self, germanData, englishData):
""" """
dfGerman = pd.DataFrame(germanData)
dfEnglish = pd.DataFrame(englishData)

with pd.ExcelWriter(self.filename, engine="openpyxl") as writer:
dfGerman.to_excel(writer, sheet_name="German", index=False)
dfEnglish.to_excel(writer, sheet_name="English", index=False)
2 changes: 2 additions & 0 deletions webcentral/src/common/serializers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ def queryset_iterator(obj, field):
.iterator()
)

if field.name == "customlink" or field.name == "customfile":
return
m2m_iter = getattr(obj, "_prefetched_objects_cache", {}).get(
field.name,
queryset_iterator(obj, field),
Expand Down
Loading

0 comments on commit 7cd9db4

Please sign in to comment.