Issue #481: Allow column import batch size to be configured

bcgov · May 31, 2024 · 287f896 · 287f896
1 parent 7fe1008
commit 287f896
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 2 deletions.
diff --git a/config/sync/bc_dc.settings.yml b/config/sync/bc_dc.settings.yml
@@ -1,3 +1,4 @@
+data_set_column_add_batch_size: 50
 data_set_review_period_alert: 30
 review_needed_message: 'Review needed.'
 review_overdue_message: 'Review overdue.'

diff --git a/html/modules/custom/bc_dc/config/install/bc_dc.settings.yml b/html/modules/custom/bc_dc/config/install/bc_dc.settings.yml
@@ -1,3 +1,4 @@
+data_set_column_add_batch_size: 50
 data_set_review_period_alert: 30
 review_needed_message: 'Review needed.'
 review_overdue_message: 'Review overdue.'
diff --git a/html/modules/custom/bc_dc/config/schema/bc_dc.schema.yml b/html/modules/custom/bc_dc/config/schema/bc_dc.schema.yml
@@ -2,6 +2,9 @@ bc_dc.settings:
   type: config_object
   label: 'BC Data Catalogue Module settings'
   mapping:
+    data_set_column_add_batch_size:
+      type: integer
+      label: 'Add columns batch size'
     data_set_review_period_alert:
       type: integer
       label: 'Review period alert'

diff --git a/html/modules/custom/bc_dc/src/Form/BcDcAddColumnsForm.php b/html/modules/custom/bc_dc/src/Form/BcDcAddColumnsForm.php
@@ -684,13 +684,14 @@ public function submitForm(array &$form, FormStateInterface $form_state): void {
     // Create paragraph entities for each row of $import_file_contents
     // using batch.
     $operations = [];
-    foreach ($import_file_contents as $column) {
+    $batch_size = $this->config('bc_dc.settings')->get('data_set_column_add_batch_size') ?? 50;
+    foreach (array_chunk($import_file_contents, $batch_size) as $columns) {
       $operations[] = [
         [static::class, 'callbackBatchOperation'],
         [
           (int) $node->id(),
           $import_file_header,
-          [$column],
+          $columns,
         ],
       ];
     }

diff --git a/html/modules/custom/bc_dc/src/Form/BcDcSettingsForm.php b/html/modules/custom/bc_dc/src/Form/BcDcSettingsForm.php
@@ -71,6 +71,15 @@ public function buildForm(array $form, FormStateInterface $form_state): array {
       '#value' => $this->t('Send metadata record review reminders'),
     ];
 
+    $form['data_set_column_add_batch_size'] = [
+      '#type' => 'number',
+      '#title' => $this->t('Add columns batch size'),
+      '#description' => $this->t('The number of columns to add in each batch when importing data columns.'),
+      '#required' => TRUE,
+      '#default_value' => $bc_dc_settings->get('data_set_column_add_batch_size'),
+      '#min' => 1,
+      '#step' => 1,
+    ];
     $form['data_set_review_period_alert'] = [
       '#type' => 'number',
       '#title' => $this->t('Review period alert'),
@@ -114,6 +123,7 @@ public function submitForm(array &$form, FormStateInterface $form_state): void {
     // Save individual config values.
     $bc_dc_settings = $this->config('bc_dc.settings');
     $fields_to_save = [
+      'data_set_column_add_batch_size',
       'data_set_review_period_alert',
       'review_needed_message',
       'review_overdue_message',