Skip to content

Commit

Permalink
Csv Upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@simon-leech
simon-leech committed Oct 31, 2024
1 parent aa57e9c commit 0eb1944
Showing 1 changed file with 150 additions and 156 deletions.
306 changes: 150 additions & 156 deletions lib/utils/csvUpload.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ This utility exports the mapp.utils.csvUpload() method to upload a CSV to a data
*/

/**
* @function csvUpload
* @function csvUpload async
* @description
* This function uploads a CSV file to a database store.
* @param {File} file - The CSV file to upload.
Expand All @@ -20,185 +20,179 @@ This utility exports the mapp.utils.csvUpload() method to upload a CSV to a data
* @property {String} [params.updateQuery] - The query to run after the data is uploaded.
* @property {Object} [params.queryparams] - The query parameters object.
* @property {Number} [params.chunkSize] - The chunk size in bytes to upload the data in. Default is 4MB.
*/

// Add dictionary definitions
mapp.utils.merge(mapp.dictionaries, {
en: {
import_csv: "Import From CSV",
number_of_columns_imported: "The number of columns in your imported file",
number_of_columns_required: "is not the same as the number of columns required",
import_successful: "Data imported successfully [Rows: ",
import_failed: "Data import failed, please try again."
}
});
* @returns {<Promise>} - The outcome object.
*/;

export default function csvUpload(file, params = {}) {
export default async function csvUpload(file, params = {}) {

// Create an outcome object that will be returned at the end of the function.
// This means other function that make use of this method can check if the import was successful.
// They can then handle the success or failure of the import and show the user a relevant message.
const outcome = {};

const reader = new FileReader();

reader.readAsText(file);

reader.onload = (e) => {
let chunkSize = 0;
let promises = [];
let chunk = [];

// Split text file into rows on carriage return / new line.
let rows = e.target.result.trim().split(/\r?\n/);

// Define the headers
let headers = rows[0].replaceAll('"', "");
// Turn headers into an array
headers = headers.split(",");
return new Promise((resolve, reject) => {
const reader = new FileReader();

// Skip the header row if flagged in config.
if (params.header) {
rows = rows.slice(1);
}
reader.readAsText(file);

// Check if headers length matches the schema length if headerCheck:true
if (
params.headerCheck &&
headers.length != params.schema.length
) {
reader.onload = (e) => {
let chunkSize = 0;
const promises = [];
let chunk = [];

// Populate the outcome object with the error message.
// This provides the calling function with a message to display to the user, about why the import failed.
outcome.error = `${mapp.dictionary.number_of_columns_imported} (${headers.length})
${mapp.dictionary.number_of_columns_required}
(${params.schema.length}). ${params.headerCheckErrorMessage || ""}`;

return;
}
// Split text file into rows on carriage return / new line.
let rows = e.target.result.trim().split(/\r?\n/);

// Define schema methods.
const schemaMethods = {
Text: (x) => `'${x.replace(/'/g, "''")}'`,
Integer: (x) => parseInt(x.replace(/[^\d.-]/g, '')) || "NULL",
Float: (x) => parseFloat(x.replace(/[^\d.-]/g, '')) || "NULL"
};
// Define the headers
let headers = rows[0].replaceAll('"', '');
// Turn headers into an array
headers = headers.split(',');

rows.forEach((_row) => {
// Split row into fields array.
let fields = _row.split(/,(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))/);
// Return if the fields array doesn't match the schema length.
if (fields.length != params.schema.length) {
return;
// Skip the header row if flagged in config.
if (params.header) {
rows = rows.slice(1);
}

// Create row string for values from by passing field value to schema method.
let row = `(${fields
.map((v, i) => schemaMethods[params.schema[i]](v))
.join()})`;

// Determine blob size of the row.
let rowSize = new Blob([row]).size;

// Check whether the chunk would exceed lambda payload limit.
// Check if headers length matches the schema length if headerCheck:true
if (
chunkSize + rowSize >=
(params.chunkSize || 1024 * 1024 * 4)
params.headerCheck &&
headers.length != params.schema.length
) {

// Push upload query into promises array.
promises.push(
mapp.utils.xhr({
method: "POST",
url:
`${mapp.host}/api/query?` +
mapp.utils.paramString({
template: params.query,
}),
body: JSON.stringify({ arr: chunk }),
})
);

// Create a new current chunk.
chunk = [];
chunkSize = 0;
// Populate the outcome object with the error message.
// This provides the calling function with a message to display to the user, about why the import failed.
outcome.error = `${mapp.dictionary.number_of_columns_imported} (${headers.length})
${mapp.dictionary.number_of_columns_required}
(${params.schema.length}). ${params.headerCheckErrorMessage || ''}`;

return;
}

// Add row to current chunk and sum size.
chunkSize += rowSize;
chunk.push(row);
});

// Push final chunk and upload query promise.
promises.push(
mapp.utils.xhr({
method: "POST",
url:
`${mapp.host}/api/query?` +
mapp.utils.paramString({
template: params.query,
}),
body: JSON.stringify({ arr: chunk }),
})
);

// Wait for all upload query promises to resolve.
Promise.all(promises).then(async (responses) => {

// Check if any of the promises errored on import
responses.forEach(element => {
if (element instanceof Error) {

// Populate the outcome object with the error message.
// This provides the calling function with a message to display to the user, about why the import failed.
outcome.error = `${mapp.dictionary.import_failed}`;
// Define schema methods.
const schemaMethods = {
Text: (x) => `'${x.replace(/'/g, '\'\'')}'`,
Integer: (x) => parseInt(x.replace(/[^\d.-]/g, '')) || 'NULL',
Float: (x) => parseFloat(x.replace(/[^\d.-]/g, '')) || 'NULL'
};

rows.forEach((_row) => {
// Split row into fields array.
const fields = _row.split(/,(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))/);
// Return if the fields array doesn't match the schema length.
if (fields.length != params.schema.length) {
return;
}

// Create row string for values from by passing field value to schema method.
const row = `(${fields
.map((v, i) => schemaMethods[params.schema[i]](v))
.join()})`;

// Determine blob size of the row.
const rowSize = new Blob([row]).size;

// Check whether the chunk would exceed lambda payload limit.
if (
chunkSize + rowSize >=
(params.chunkSize || 1024 * 1024 * 4)
) {

// Push upload query into promises array.
promises.push(
mapp.utils.xhr({
method: 'POST',
url:
`${mapp.host}/api/query?` +
mapp.utils.paramString({
template: params.query,
}),
body: JSON.stringify({ arr: chunk }),
})
);

// Create a new current chunk.
chunk = [];
chunkSize = 0;
}

// Add row to current chunk and sum size.
chunkSize += rowSize;
chunk.push(row);
});

// An updateQuery is ran once all the import promises have resolved.
if (params.updateQuery) {
// Create an object to store the query and queryparams.
const queryObject = {};
queryObject.queryparams ??= params.queryparams || {};
queryObject.query = params.updateQuery;
const paramString = mapp.utils.paramString(mapp.utils.queryParams(entry));

// Execute updateQuery
await mapp.utils.xhr(`${mapp.host} /api/query ? ${paramString} `)
.then((response) => {

// Check if the response is an error
if (response instanceof Error) {

// Populate the outcome object with the error message.
// This provides the calling function with a message to display to the user, about why the import failed.
outcome.error = `${mapp.dictionary.import_failed}`;
return;
}

// If the response is not null, its possible the database returned a response.
if (response !== null) {
// We need to check if the database returns a response
const keys = Object.keys(response);
if (keys.length > 0) {
// Get the value from the first key.
const firstKey = keys[0];
const value = response[firstKey];

if (value) {
// Populate the outcome object with the error message.
// This provides the calling function with a message to display to the user
// This response is returned from the database.
outcome.error = `${value}`;
// Push final chunk and upload query promise.
promises.push(
mapp.utils.xhr({
method: 'POST',
url:
`${mapp.host}/api/query?` +
mapp.utils.paramString({
template: params.query,
}),
body: JSON.stringify({ arr: chunk }),
})
);

// Wait for all upload query promises to resolve.
Promise.all(promises).then(async (responses) => {

// Check if any of the promises errored on import
responses.forEach(element => {
if (element instanceof Error) {

// Populate the outcome object with the error message.
// This provides the calling function with a message to display to the user, about why the import failed.
outcome.error = `${mapp.dictionary.import_failed}`;
return;
}
});

// An updateQuery is ran once all the import promises have resolved.
if (params.updateQuery) {
// Create an object to store the query and queryparams.
const queryObject = {};
queryObject.queryparams ??= params.queryparams || {};
queryObject.query = params.updateQuery;
const paramString = mapp.utils.paramString(mapp.utils.queryParams(queryObject));

// Execute updateQuery
await mapp.utils.xhr(`${mapp.host}/api/query?${paramString} `)
.then((response) => {

// Check if the response is an error
if (response instanceof Error) {

// Populate the outcome object with the error message.
// This provides the calling function with a message to display to the user, about why the import failed.
outcome.error = `${mapp.dictionary.import_failed}`;
return;
}

// If the response is not null, its possible the database returned a response.
if (response !== null) {
// We need to check if the database returns a response
const keys = Object.keys(response);
if (keys.length > 0) {
// Get the value from the first key.
const firstKey = keys[0];
const value = response[firstKey];

if (value) {
// Populate the outcome object with the error message.
// This provides the calling function with a message to display to the user
// This response is returned from the database.
outcome.error = `${value}`;
}
}
}
}
})
}
})
}

// Return the outcome variable to the calling function.
return outcome;
});
}
resolve(outcome); // Resolve with outcome at the end
}).catch(error => {
outcome.error = error.message;
resolve(outcome);
});
};
});
}

0 comments on commit 0eb1944

Please sign in to comment.