From bcf5e28e7e3a15257ceb153ecd57d56a9376dc74 Mon Sep 17 00:00:00 2001 From: "chafique.delli" Date: Thu, 23 Jun 2022 15:42:28 +0200 Subject: [PATCH] [FIX] error messages do not appear in the excel file during import and chunks are not split correctly --- pattern_import_export/models/pattern_chunk.py | 10 +++++++++- pattern_import_export/models/pattern_config.py | 6 +++++- pattern_import_export/models/pattern_file.py | 6 ++++-- pattern_import_export_xlsx/models/__init__.py | 1 + .../models/pattern_chunk.py | 16 ++++++++++++++++ .../models/pattern_config.py | 3 ++- .../models/pattern_file.py | 13 +++++++------ 7 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 pattern_import_export_xlsx/models/pattern_chunk.py diff --git a/pattern_import_export/models/pattern_chunk.py b/pattern_import_export/models/pattern_chunk.py index 23c0947b..842f2e95 100644 --- a/pattern_import_export/models/pattern_chunk.py +++ b/pattern_import_export/models/pattern_chunk.py @@ -73,7 +73,15 @@ def run(self): "state": "failed", } ) - self.with_delay().check_last() + if not self.messages: + self.messages = {"message": e.args[0]} + next_chunk = self.get_next_chunk() + if next_chunk: + config = self.pattern_file_id.pattern_config_id + priority = config.job_priority + next_chunk.with_delay(priority=priority).run() + else: + self.with_delay().check_last() return "OK" def _prepare_chunk_result(self, res): diff --git a/pattern_import_export/models/pattern_config.py b/pattern_import_export/models/pattern_config.py index 74d025f3..64e2f68a 100644 --- a/pattern_import_export/models/pattern_config.py +++ b/pattern_import_export/models/pattern_config.py @@ -128,6 +128,7 @@ def _get_header(self, use_description=False): header = [] for export_line in self.export_fields: header.extend(export_line._get_header(use_description)) + header.insert(0, "#Error") return header def generate_pattern(self): @@ -173,7 +174,10 @@ def json2pattern_format(self, data): key = key.replace(IDENTIFIER_SUFFIX, "") if key == ".id": key = "id" - val = val[key] + if key == "#Error": + val = None + else: + val = val[key] if val is None: break except IndexError: diff --git a/pattern_import_export/models/pattern_file.py b/pattern_import_export/models/pattern_file.py index ccbe4879..aa5eac7d 100644 --- a/pattern_import_export/models/pattern_file.py +++ b/pattern_import_export/models/pattern_file.py @@ -39,6 +39,8 @@ def _compute_stat(self): record.progress = (record.nbr_error + record.nbr_success) * 100.0 / todo else: record.progress = 0 + if record.nbr_error and record.state != "failed": + record.state = "failed" @api.model_create_multi def create(self, vals): @@ -141,7 +143,7 @@ def _prepare_chunk(self, start_idx, stop_idx, data): def _should_create_chunk(self, items, next_item): """Customise this code if you want to add some additionnal item after reaching the limit""" - return len(items) > self.pattern_config_id.chunk_size + return len(items) >= self.pattern_config_id.chunk_size def _create_chunk(self, start_idx, stop_idx, data): vals = self._prepare_chunk(start_idx, stop_idx, data) @@ -157,7 +159,7 @@ def split_in_chunk(self): self.chunk_ids.unlink() try: items = [] - start_idx = 1 + start_idx = self.pattern_config_id.nr_of_header_rows + 1 previous_idx = None # idx is the index position in the original file # we can have empty line that can be skipped diff --git a/pattern_import_export_xlsx/models/__init__.py b/pattern_import_export_xlsx/models/__init__.py index 4dbee548..e63a92ca 100644 --- a/pattern_import_export_xlsx/models/__init__.py +++ b/pattern_import_export_xlsx/models/__init__.py @@ -1,2 +1,3 @@ from . import pattern_config from . import pattern_file +from . import pattern_chunk diff --git a/pattern_import_export_xlsx/models/pattern_chunk.py b/pattern_import_export_xlsx/models/pattern_chunk.py new file mode 100644 index 00000000..35059c2c --- /dev/null +++ b/pattern_import_export_xlsx/models/pattern_chunk.py @@ -0,0 +1,16 @@ +# Copyright 2022 Akretion (https://www.akretion.com). +# @author Chafique DELLI +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo import models + + +class PatternChunk(models.Model): + _inherit = "pattern.chunk" + + def check_last(self): + res = super().check_last() + if res == "There is still some running chunk": + for record in self: + record.pattern_file_id.write_error_in_xlsx() + return res diff --git a/pattern_import_export_xlsx/models/pattern_config.py b/pattern_import_export_xlsx/models/pattern_config.py index 2cb4c8cd..33fba0b4 100644 --- a/pattern_import_export_xlsx/models/pattern_config.py +++ b/pattern_import_export_xlsx/models/pattern_config.py @@ -43,6 +43,7 @@ def _build_main_sheet_structure(self, book): for row, lines in enumerate(self._get_output_headers(), start=1): for col, header in enumerate(lines.values(), start=1): main_sheet.cell(row=row, column=col, value=header) + main_sheet.column_dimensions["A"].hidden = True return main_sheet def _populate_main_sheet_rows(self, main_sheet, records): @@ -83,7 +84,7 @@ def _create_validators(self, main_sheet, records, tabs): formula_range_src = "=" + quote_sheetname(tab_name) + "!" + range_src validation = DataValidation(type="list", formula1=formula_range_src) for idx_col in tab["idx_col_validator"]: - col_letter_dst = get_column_letter(idx_col) + col_letter_dst = get_column_letter(idx_col + 1) range_dst = "${}${}:${}${}".format( col_letter_dst, str(self.row_start_records), diff --git a/pattern_import_export_xlsx/models/pattern_file.py b/pattern_import_export_xlsx/models/pattern_file.py index 680528ea..48aa4a1d 100644 --- a/pattern_import_export_xlsx/models/pattern_file.py +++ b/pattern_import_export_xlsx/models/pattern_file.py @@ -70,23 +70,24 @@ def write_error_in_xlsx(self): infile = BytesIO(base64.b64decode(self.datas)) wb = openpyxl.load_workbook(filename=infile) ws = self._get_worksheet(wb) + ws.column_dimensions["A"].hidden = False # we clear the error col if exist if ws["A1"].value == _("#Error"): ws.delete_cols(1) ws.insert_cols(1) ws.cell(1, 1, value=_("#Error")) - last_row_idx = 0 - for chunk in self.chunk_ids: + for chunk in self.chunk_ids.filtered(lambda c: c.nbr_error > 0): for message in chunk.messages: if "rows" in message: - last_row_idx = message["rows"]["to"] - ws.cell(message["rows"]["to"], 1, value=message["message"].strip()) + ws.cell(message["rows"]["from"], 1, value=message["message"]) else: # If no row are specify, this is a global message # that should be applied until the end of the chunk - for idx in range(last_row_idx, chunk.stop_idx + 1): - ws.cell(idx, 1, value=message["message"].strip()) + for idx in range(chunk.start_idx, chunk.stop_idx + 1): + if self.pattern_config_id.nr_of_header_rows == 2 and idx == 2: + idx = idx + 1 + ws.cell(idx, 1, value=message["message"]) output = BytesIO() wb.save(output) self.datas = base64.b64encode(output.getvalue())