diff --git a/lib/utils/csvUpload.mjs b/lib/utils/csvUpload.mjs index a57494b56..cb45d7246 100644 --- a/lib/utils/csvUpload.mjs +++ b/lib/utils/csvUpload.mjs @@ -7,7 +7,7 @@ This utility exports the mapp.utils.csvUpload() method to upload a CSV to a data */ /** - * @function csvUpload + * @function csvUpload async * @description * This function uploads a CSV file to a database store. * @param {File} file - The CSV file to upload. @@ -20,185 +20,179 @@ This utility exports the mapp.utils.csvUpload() method to upload a CSV to a data * @property {String} [params.updateQuery] - The query to run after the data is uploaded. * @property {Object} [params.queryparams] - The query parameters object. * @property {Number} [params.chunkSize] - The chunk size in bytes to upload the data in. Default is 4MB. -*/ - -// Add dictionary definitions -mapp.utils.merge(mapp.dictionaries, { - en: { - import_csv: "Import From CSV", - number_of_columns_imported: "The number of columns in your imported file", - number_of_columns_required: "is not the same as the number of columns required", - import_successful: "Data imported successfully [Rows: ", - import_failed: "Data import failed, please try again." - } -}); + * @returns {} - The outcome object. +*/; -export default function csvUpload(file, params = {}) { +export default async function csvUpload(file, params = {}) { // Create an outcome object that will be returned at the end of the function. // This means other function that make use of this method can check if the import was successful. // They can then handle the success or failure of the import and show the user a relevant message. const outcome = {}; - const reader = new FileReader(); - - reader.readAsText(file); - - reader.onload = (e) => { - let chunkSize = 0; - let promises = []; - let chunk = []; - - // Split text file into rows on carriage return / new line. - let rows = e.target.result.trim().split(/\r?\n/); - - // Define the headers - let headers = rows[0].replaceAll('"', ""); - // Turn headers into an array - headers = headers.split(","); + return new Promise((resolve, reject) => { + const reader = new FileReader(); - // Skip the header row if flagged in config. - if (params.header) { - rows = rows.slice(1); - } + reader.readAsText(file); - // Check if headers length matches the schema length if headerCheck:true - if ( - params.headerCheck && - headers.length != params.schema.length - ) { + reader.onload = (e) => { + let chunkSize = 0; + const promises = []; + let chunk = []; - // Populate the outcome object with the error message. - // This provides the calling function with a message to display to the user, about why the import failed. - outcome.error = `${mapp.dictionary.number_of_columns_imported} (${headers.length}) - ${mapp.dictionary.number_of_columns_required} - (${params.schema.length}). ${params.headerCheckErrorMessage || ""}`; - - return; - } + // Split text file into rows on carriage return / new line. + let rows = e.target.result.trim().split(/\r?\n/); - // Define schema methods. - const schemaMethods = { - Text: (x) => `'${x.replace(/'/g, "''")}'`, - Integer: (x) => parseInt(x.replace(/[^\d.-]/g, '')) || "NULL", - Float: (x) => parseFloat(x.replace(/[^\d.-]/g, '')) || "NULL" - }; + // Define the headers + let headers = rows[0].replaceAll('"', ''); + // Turn headers into an array + headers = headers.split(','); - rows.forEach((_row) => { - // Split row into fields array. - let fields = _row.split(/,(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))/); - // Return if the fields array doesn't match the schema length. - if (fields.length != params.schema.length) { - return; + // Skip the header row if flagged in config. + if (params.header) { + rows = rows.slice(1); } - // Create row string for values from by passing field value to schema method. - let row = `(${fields - .map((v, i) => schemaMethods[params.schema[i]](v)) - .join()})`; - - // Determine blob size of the row. - let rowSize = new Blob([row]).size; - - // Check whether the chunk would exceed lambda payload limit. + // Check if headers length matches the schema length if headerCheck:true if ( - chunkSize + rowSize >= - (params.chunkSize || 1024 * 1024 * 4) + params.headerCheck && + headers.length != params.schema.length ) { - // Push upload query into promises array. - promises.push( - mapp.utils.xhr({ - method: "POST", - url: - `${mapp.host}/api/query?` + - mapp.utils.paramString({ - template: params.query, - }), - body: JSON.stringify({ arr: chunk }), - }) - ); - - // Create a new current chunk. - chunk = []; - chunkSize = 0; + // Populate the outcome object with the error message. + // This provides the calling function with a message to display to the user, about why the import failed. + outcome.error = `${mapp.dictionary.number_of_columns_imported} (${headers.length}) + ${mapp.dictionary.number_of_columns_required} + (${params.schema.length}). ${params.headerCheckErrorMessage || ''}`; + + return; } - // Add row to current chunk and sum size. - chunkSize += rowSize; - chunk.push(row); - }); - - // Push final chunk and upload query promise. - promises.push( - mapp.utils.xhr({ - method: "POST", - url: - `${mapp.host}/api/query?` + - mapp.utils.paramString({ - template: params.query, - }), - body: JSON.stringify({ arr: chunk }), - }) - ); - - // Wait for all upload query promises to resolve. - Promise.all(promises).then(async (responses) => { - - // Check if any of the promises errored on import - responses.forEach(element => { - if (element instanceof Error) { - - // Populate the outcome object with the error message. - // This provides the calling function with a message to display to the user, about why the import failed. - outcome.error = `${mapp.dictionary.import_failed}`; + // Define schema methods. + const schemaMethods = { + Text: (x) => `'${x.replace(/'/g, '\'\'')}'`, + Integer: (x) => parseInt(x.replace(/[^\d.-]/g, '')) || 'NULL', + Float: (x) => parseFloat(x.replace(/[^\d.-]/g, '')) || 'NULL' + }; + + rows.forEach((_row) => { + // Split row into fields array. + const fields = _row.split(/,(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))/); + // Return if the fields array doesn't match the schema length. + if (fields.length != params.schema.length) { return; } + + // Create row string for values from by passing field value to schema method. + const row = `(${fields + .map((v, i) => schemaMethods[params.schema[i]](v)) + .join()})`; + + // Determine blob size of the row. + const rowSize = new Blob([row]).size; + + // Check whether the chunk would exceed lambda payload limit. + if ( + chunkSize + rowSize >= + (params.chunkSize || 1024 * 1024 * 4) + ) { + + // Push upload query into promises array. + promises.push( + mapp.utils.xhr({ + method: 'POST', + url: + `${mapp.host}/api/query?` + + mapp.utils.paramString({ + template: params.query, + }), + body: JSON.stringify({ arr: chunk }), + }) + ); + + // Create a new current chunk. + chunk = []; + chunkSize = 0; + } + + // Add row to current chunk and sum size. + chunkSize += rowSize; + chunk.push(row); }); - // An updateQuery is ran once all the import promises have resolved. - if (params.updateQuery) { - // Create an object to store the query and queryparams. - const queryObject = {}; - queryObject.queryparams ??= params.queryparams || {}; - queryObject.query = params.updateQuery; - const paramString = mapp.utils.paramString(mapp.utils.queryParams(entry)); - - // Execute updateQuery - await mapp.utils.xhr(`${mapp.host} /api/query ? ${paramString} `) - .then((response) => { - - // Check if the response is an error - if (response instanceof Error) { - - // Populate the outcome object with the error message. - // This provides the calling function with a message to display to the user, about why the import failed. - outcome.error = `${mapp.dictionary.import_failed}`; - return; - } - - // If the response is not null, its possible the database returned a response. - if (response !== null) { - // We need to check if the database returns a response - const keys = Object.keys(response); - if (keys.length > 0) { - // Get the value from the first key. - const firstKey = keys[0]; - const value = response[firstKey]; - - if (value) { - // Populate the outcome object with the error message. - // This provides the calling function with a message to display to the user - // This response is returned from the database. - outcome.error = `${value}`; + // Push final chunk and upload query promise. + promises.push( + mapp.utils.xhr({ + method: 'POST', + url: + `${mapp.host}/api/query?` + + mapp.utils.paramString({ + template: params.query, + }), + body: JSON.stringify({ arr: chunk }), + }) + ); + + // Wait for all upload query promises to resolve. + Promise.all(promises).then(async (responses) => { + + // Check if any of the promises errored on import + responses.forEach(element => { + if (element instanceof Error) { + + // Populate the outcome object with the error message. + // This provides the calling function with a message to display to the user, about why the import failed. + outcome.error = `${mapp.dictionary.import_failed}`; + return; + } + }); + + // An updateQuery is ran once all the import promises have resolved. + if (params.updateQuery) { + // Create an object to store the query and queryparams. + const queryObject = {}; + queryObject.queryparams ??= params.queryparams || {}; + queryObject.query = params.updateQuery; + const paramString = mapp.utils.paramString(mapp.utils.queryParams(queryObject)); + + // Execute updateQuery + await mapp.utils.xhr(`${mapp.host}/api/query?${paramString} `) + .then((response) => { + + // Check if the response is an error + if (response instanceof Error) { + + // Populate the outcome object with the error message. + // This provides the calling function with a message to display to the user, about why the import failed. + outcome.error = `${mapp.dictionary.import_failed}`; + return; + } + + // If the response is not null, its possible the database returned a response. + if (response !== null) { + // We need to check if the database returns a response + const keys = Object.keys(response); + if (keys.length > 0) { + // Get the value from the first key. + const firstKey = keys[0]; + const value = response[firstKey]; + + if (value) { + // Populate the outcome object with the error message. + // This provides the calling function with a message to display to the user + // This response is returned from the database. + outcome.error = `${value}`; + } } } - } - }) - } + }) + } - // Return the outcome variable to the calling function. - return outcome; - }); - } + resolve(outcome); // Resolve with outcome at the end + }).catch(error => { + outcome.error = error.message; + resolve(outcome); + }); + }; + }); } \ No newline at end of file