Merge pull request #5204 from specify/issue-2959

Add preference to add UTF-8 BOM to CSV exports (For Excel)
specify · Aug 15, 2024 · 4300ff2 · 4300ff2
2 parents ba5f51f + 20c77e4
commit 4300ff2
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 8 deletions.
diff --git a/specifyweb/frontend/js_src/lib/components/Preferences/UserDefinitions.tsx b/specifyweb/frontend/js_src/lib/components/Preferences/UserDefinitions.tsx
@@ -1572,6 +1572,16 @@ export const userPreferenceDefinitions = {
               },
             ],
           }),
+          exportCsvUtf8Bom: definePref<boolean>({
+            title: preferencesText.exportCsvUtf8Bom(),
+            description: (
+              <span>{preferencesText.exportCsvUtf8BomDescription()}</span>
+            ),
+            requiresReload: false,
+            visible: true,
+            defaultValue: true,
+            type: 'java.lang.Boolean',
+          }),
           displayBasicView: definePref<boolean>({
             title: preferencesText.displayBasicView(),
             requiresReload: false,

diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Export.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Export.tsx
@@ -51,7 +51,11 @@ export function QueryExportButtons({
     undefined
   );
 
-  function doQueryExport(url: string, delimiter: string | undefined): void {
+  function doQueryExport(
+    url: string,
+    delimiter: string | undefined,
+    bom: boolean | undefined
+  ): void {
     if (typeof getQueryFieldRecords === 'function')
       queryResource.set('fields', getQueryFieldRecords());
     const serialized = queryResource.toJSON();
@@ -67,6 +71,7 @@ export function QueryExportButtons({
           ),
         recordSetId,
         delimiter,
+        bom,
       }),
       errorMode: 'dismissible',
     });
@@ -78,6 +83,12 @@ export function QueryExportButtons({
     'exportFileDelimiter'
   );
 
+  const [utf8Bom] = userPreferences.use(
+    'queryBuilder',
+    'behavior',
+    'exportCsvUtf8Bom'
+  );
+
   /*
    *Will be only called if query is not distinct,
    *selection not enabled when distinct selected
@@ -105,7 +116,13 @@ export function QueryExportButtons({
         generateMappingPathPreview(baseTableName, field.mappingPath)
       );
 
-    return downloadDataSet(name, filteredResults, columnsName, separator);
+    return downloadDataSet(
+      name,
+      filteredResults,
+      columnsName,
+      separator,
+      utf8Bom
+    );
   }
 
   const containsResults = results.current?.some((row) => row !== undefined);
@@ -141,7 +158,7 @@ export function QueryExportButtons({
           showConfirmation={showConfirmation}
           onClick={(): void => {
             selectedRows.size === 0
-              ? doQueryExport('/stored_query/exportcsv/', separator)
+              ? doQueryExport('/stored_query/exportcsv/', separator, utf8Bom)
               : exportSelected().catch(softFail);
           }}
         >
@@ -154,7 +171,7 @@ export function QueryExportButtons({
           showConfirmation={showConfirmation}
           onClick={(): void =>
             hasLocalityColumns(fields)
-              ? doQueryExport('/stored_query/exportkml/', undefined)
+              ? doQueryExport('/stored_query/exportkml/', undefined, undefined)
               : setState('warning')
           }
         >

diff --git a/specifyweb/frontend/js_src/lib/components/WorkBench/helpers.ts b/specifyweb/frontend/js_src/lib/components/WorkBench/helpers.ts
@@ -7,13 +7,15 @@ export const downloadDataSet = async (
   name: string,
   rows: RA<RA<string>>,
   columns: RA<string>,
-  delimiter: string
+  delimiter: string,
+  bom: boolean = false
 ): Promise<void> =>
   new Promise((resolve, reject) =>
     stringify(
       [columns, ...rows],
       {
         delimiter,
+        bom,
       },
       (error, output) => {
         if (error === undefined)

diff --git a/specifyweb/frontend/js_src/lib/localization/preferences.ts b/specifyweb/frontend/js_src/lib/localization/preferences.ts
@@ -1060,6 +1060,26 @@ export const preferencesText = createDictionary({
     'uk-ua': 'Роздільник файлу експорту',
     'de-ch': 'Trennzeichen für Exportdateien',
   },
+  exportCsvUtf8Bom: {
+    'en-us': 'Add UTF-8 BOM to CSV file exports',
+    'ru-ru': 'Добавить UTF-8 BOM в экспорт CSV-файла',
+    'es-es': 'Agregar BOM UTF-8 a las exportaciones de archivos CSV',
+    'fr-fr': 'Ajouter UTF-8 BOM aux exportations de fichiers CSV',
+    'uk-ua': 'Додайте специфікацію UTF-8 до експорту файлу CSVу',
+    'de-ch': 'UTF-8 BOM zum CSV-Dateiexport hinzufügen',
+  },
+  exportCsvUtf8BomDescription: {
+    'en-us':
+      'Adds a BOM (Byte Order Mark) to exported CSV files to ensure that the file is correctly recognized and displayed by various programs (Excel, OpenRefine, etc.), preventing issues with special characters and formatting.',
+    'ru-ru': 'Корректное отображение экспортированных CSV-файлов в Excel.',
+    'es-es':
+      'Hace que las exportaciones de archivos CSV se muestren correctamente en Excel.',
+    'fr-fr':
+      "Permet aux exportations de fichiers CSV de s'afficher correctement dans Excel.",
+    'uk-ua': 'Змушує експорт файлів CSV правильно відображатися в Excel.',
+    'de-ch':
+      'Sorgt dafür, dass CSV-Dateiexporte in Excel korrekt angezeigt werden.',
+  },
   caseSensitive: {
     'en-us': 'Case-sensitive',
     'ru-ru': 'С учетом регистра',

diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py
@@ -158,7 +158,7 @@ def do_export(spquery, collection, user, filename, exporttype, host):
             query_to_csv(session, collection, user, tableid, field_specs, path,
                          recordsetid=recordsetid, 
                          captions=spquery['captions'], strip_id=True,
-                         distinct=spquery['selectdistinct'], delimiter=spquery['delimiter'],)
+                         distinct=spquery['selectdistinct'], delimiter=spquery['delimiter'], bom=spquery['bom'])
         elif exporttype == 'kml':
             query_to_kml(session, collection, user, tableid, field_specs, path, spquery['captions'], host,
                          recordsetid=recordsetid, strip_id=False)
@@ -186,7 +186,7 @@ def stored_query_to_csv(query_id, collection, user, path):
 
 def query_to_csv(session, collection, user, tableid, field_specs, path,
                  recordsetid=None, captions=False, strip_id=False, row_filter=None,
-                 distinct=False, delimiter=','):
+                 distinct=False, delimiter=',', bom=False):
     """Build a sqlalchemy query using the QueryField objects given by
     field_specs and send the results to a CSV file at the given
     file path.
@@ -198,7 +198,11 @@ def query_to_csv(session, collection, user, tableid, field_specs, path,
 
     logger.debug('query_to_csv starting')
 
-    with open(path, 'w', newline='', encoding='utf-8') as f:
+    encoding = 'utf-8'
+    if bom:
+        encoding = 'utf-8-sig'
+
+    with open(path, 'w', newline='', encoding=encoding) as f:
         csv_writer = csv.writer(f, delimiter=delimiter)
         if captions:
             header = captions