Skip to content

Commit

Permalink
Merge pull request #11 from data-catering/override-connection-details
Browse files Browse the repository at this point in the history 
Add in ability to define additional connection options when creating plan
  • Loading branch information
@pflooky
pflooky authored Apr 23, 2024
2 parents 726c070 + e2f05aa commit 83c3e5a
Show file tree
Hide file tree
Showing 11 changed files with 197 additions and 64 deletions.
19 changes: 19 additions & 0 deletions app/src/main/resources/ui/configuration-data.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1252,3 +1252,22 @@ dataSourcePropertiesMap.set("slack", {
}
}
});

export let subDataSourceConfigMap = new Map();
subDataSourceConfigMap.set("http", {
method: {
displayName: "Method",
default: "",
type: "text",
help: "HTTP method.",
choice: ["", "GET", "POST", "PUT", "DELETE", "PATCH", "CONNECT", "OPTIONS", "TRACE", "HEAD"],
override: "true"
},
endpoint: {
displayName: "Endpoint",
default: "",
type: "text",
help: "Endpoint pathway (i.e. '/my-path/data').",
override: "true"
}
});
67 changes: 52 additions & 15 deletions app/src/main/resources/ui/helper-foreign-keys.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,17 @@ Ability to choose task name and columns. Define custom relationships.
*/
import {
addAccordionCloseButton,
addConnectionOverrideOptions,
createAccordionItem,
createButton,
createCloseButton,
createFieldValidationCheck,
createFormFloating,
createInput,
createSelect,
createTooltip,
dispatchEvent,
getOverrideConnectionOptionsAsMap,
wait
} from "./shared.js";

Expand Down Expand Up @@ -47,8 +51,10 @@ async function createForeignKeyLinksFromPlan(newForeignKey, foreignKey, linkType
for (const fkLink of Array.from(foreignKey[`${linkType}Links`])) {
let newForeignKeyLink = await createForeignKeyInput(numForeignKeysLinks, `foreign-key-${linkType}-link`);
foreignKeyLinkSources.insertBefore(newForeignKeyLink, foreignKeyLinkSources.lastChild);
$(newForeignKeyLink).find(`select.foreign-key-${linkType}-link`).selectpicker("val", fkLink.taskName);
$(newForeignKeyLink).find(`input.foreign-key-${linkType}-link`).val(fkLink.columns)[0].dispatchEvent(new Event("input"));
let updatedForeignKeyTaskName = $(newForeignKeyLink).find(`select.foreign-key-${linkType}-link`).selectpicker("val", fkLink.taskName);
dispatchEvent(updatedForeignKeyTaskName, "change");
let updatedForeignKeyColumns = $(newForeignKeyLink).find(`input.foreign-key-${linkType}-link`).val(fkLink.columns);
dispatchEvent(updatedForeignKeyColumns, "input");
}
}

Expand All @@ -61,8 +67,10 @@ export async function createForeignKeysFromPlan(respJson) {
foreignKeysAccordion.append(newForeignKey);

if (foreignKey.source) {
$(newForeignKey).find("select.foreign-key-source").selectpicker("val", foreignKey.source.taskName);
$(newForeignKey).find("input.foreign-key-source").val(foreignKey.source.columns)[0].dispatchEvent(new Event("input"));
let updatedTaskName = $(newForeignKey).find("select.foreign-key-source").selectpicker("val", foreignKey.source.taskName);
dispatchEvent(updatedTaskName, "change");
let updatedColumns = $(newForeignKey).find("input.foreign-key-source").val(foreignKey.source.columns);
dispatchEvent(updatedColumns, "input");
}

if (foreignKey.generationLinks) {
Expand All @@ -75,23 +83,24 @@ export async function createForeignKeysFromPlan(respJson) {
}
}

function getForeignKeyLinksToArray(fkContainer, className) {
let fkGenerationLinks = $(fkContainer).find(className);
let fkGenerationLinkArray = [];
for (let fkLink of fkGenerationLinks) {
let fkLinkDetails = getForeignKeyDetail(fkLink);
fkGenerationLinkArray.push(fkLinkDetails);
function getForeignKeyLinksToArray(foreignKeyContainer, className) {
let mainContainer = $(foreignKeyContainer).find(className);
let foreignKeyLinks = $(mainContainer).find(".foreign-key-input-container");
let foreignKeyLinksArray = [];
for (let foreignKeyLink of foreignKeyLinks) {
let foreignKeyLinkDetails = getForeignKeyDetail(foreignKeyLink);
foreignKeyLinksArray.push(foreignKeyLinkDetails);
}
return fkGenerationLinkArray;
return foreignKeyLinksArray;
}

export function getForeignKeys() {
let foreignKeyContainers = Array.from(document.querySelectorAll(".foreign-key-container").values());
return foreignKeyContainers.map(fkContainer => {
let fkSource = $(fkContainer).find(".foreign-key-main-source");
let fkSourceDetails = getForeignKeyDetail(fkSource[0]);
let fkGenerationLinkArray = getForeignKeyLinksToArray(fkContainer, ".foreign-key-generation-link-source");
let fkDeleteLinkArray = getForeignKeyLinksToArray(fkContainer, ".foreign-key-delete-link-source");
let fkGenerationLinkArray = getForeignKeyLinksToArray(fkContainer, ".foreign-key-generation-link-sources");
let fkDeleteLinkArray = getForeignKeyLinksToArray(fkContainer, ".foreign-key-delete-link-sources");
return {source: fkSourceDetails, generationLinks: fkGenerationLinkArray, deleteLinks: fkDeleteLinkArray};
});
}
Expand All @@ -106,6 +115,10 @@ async function createForeignKeyLinks(index, linkType) {
let addLinkForeignKeyButton = createButton(`add-foreign-key-${linkType}-link-btn-${index}`, "add-link", "btn btn-secondary", "+ Link");
addLinkForeignKeyButton.addEventListener("click", async function () {
numForeignKeysLinks += 1;
if (linkSourceForeignKeys.childElementCount > 1) {
let divider = document.createElement("hr");
linkSourceForeignKeys.insertBefore(divider, addLinkForeignKeyButton);
}
let newForeignKeyLink = await createForeignKeyInput(numForeignKeysLinks, `foreign-key-${linkType}-link`);
linkSourceForeignKeys.insertBefore(newForeignKeyLink, addLinkForeignKeyButton);
});
Expand Down Expand Up @@ -161,6 +174,8 @@ async function updateForeignKeyTasks(taskNameSelect) {
}

async function createForeignKeyInput(index, name) {
let foreignKeyContainer = document.createElement("div");
foreignKeyContainer.setAttribute("class", "foreign-key-input-container m-1");
let foreignKey = document.createElement("div");
foreignKey.setAttribute("class", `row m-1 align-items-center ${name}-source`);
// input is task name -> column(s)
Expand All @@ -175,6 +190,22 @@ async function createForeignKeyInput(index, name) {
let columnNameFloating = createFormFloating("Column(s)", columnNamesInput);

foreignKey.append(taskNameCol, columnNameFloating);
//when task name is selected, offer input to define sub data source if not defined
//(i.e. schema and table for Postgres task with no schema and table defined, only offer table if schema is defined in data source)
//for a http data source, endpoint is not part of the data source
//same logic can be shared for data generation/validation to allow re-use of connection
let iconDiv = createTooltip();
let overrideOptionsContainer = document.createElement("div");
overrideOptionsContainer.setAttribute("class", "foreign-key-connection-container");
taskNameSelect.addEventListener("change", (event) => {
let taskName = event.target.value;
//get the corresponding task data source connection name
let taskNameInput = $(document).find(`input[class~=task-name-field]`).filter(function () {
return this.value === taskName
});
let connectionName = $(taskNameInput).closest("[class~=row]").find("select[class~=data-connection-name]").val();
addConnectionOverrideOptions(connectionName, iconDiv, overrideOptionsContainer, "foreign-key-connection-property", index);
});
if (name === "foreign-key-generation-link" || name === "foreign-key-delete-link") {
let closeButton = createCloseButton(foreignKey);
foreignKey.append(closeButton);
Expand All @@ -188,11 +219,17 @@ async function createForeignKeyInput(index, name) {
updateForeignKeyTasks(taskNameSelect);
});
await updateForeignKeyTasks(taskNameSelect);
return foreignKey;
foreignKeyContainer.append(foreignKey, overrideOptionsContainer);
return foreignKeyContainer;
}

function getForeignKeyDetail(element) {
let taskName = $(element).find("select[aria-label=Task]").val();
let columns = $(element).find("input[aria-label=Columns]").val();
return {taskName: taskName, columns: columns};
let baseForeignKey = {taskName: taskName, columns: columns};
let overrideConnectionOptions = getOverrideConnectionOptionsAsMap(element);
if (Object.keys(overrideConnectionOptions).length > 0) {
baseForeignKey["options"] = overrideConnectionOptions;
}
return baseForeignKey;
}
10 changes: 7 additions & 3 deletions app/src/main/resources/ui/helper-generation.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
createNewField,
createSelect,
createToast,
dispatchEvent,
findNextLevelNodesByClass,
getDataConnectionsAndAddToSelect,
wait
Expand Down Expand Up @@ -90,8 +91,10 @@ async function createGenerationFields(dataSourceFields, manualSchema) {
numFields += 1;
let newField = await createNewField(numFields, "generation");
$(manualSchema).find(".accordion").first().append(newField);
$(newField).find("[id^=field-name]").val(field.name)[0].dispatchEvent(new Event("input"));
$(newField).find("select[class~=field-type]").selectpicker("val", field.type)[0].dispatchEvent(new Event("change"));
let updatedFieldName = $(newField).find("[id^=field-name]").val(field.name);
dispatchEvent(updatedFieldName, "input");
let updatedFieldType = $(newField).find("select[class~=field-type]").selectpicker("val", field.type);
dispatchEvent(updatedFieldType, "change");

if (field.options) {
for (const [optKey, optVal] of Object.entries(field.options)) {
Expand All @@ -115,7 +118,8 @@ async function createGenerationFields(dataSourceFields, manualSchema) {
}

async function createAutoGenerationSchema(autoFromMetadataSchema, dataSource) {
$(autoFromMetadataSchema).find(".metadata-connection-name").selectpicker("val", dataSource.fields.optMetadataSource.name)[0].dispatchEvent(new Event("change"));
let updatedMetadataSource = $(autoFromMetadataSchema).find(".metadata-connection-name").selectpicker("val", dataSource.fields.optMetadataSource.name);
dispatchEvent(updatedMetadataSource, "change");
// takes some time to get the override options
await wait(100);
for (let [key, value] of Object.entries(dataSource.fields.optMetadataSource.overrideOptions)) {
Expand Down
6 changes: 4 additions & 2 deletions app/src/main/resources/ui/helper-record-count.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ import {
createFormText,
createInput,
createRadioButtons,
createSelect
createSelect,
dispatchEvent
} from "./shared.js";


Expand Down Expand Up @@ -66,7 +67,8 @@ export function createCountElementsFromPlan(dataSource, newDataSource) {
$(newDataSource).find("[id^=per-column-record-count]").val(dsCount.perColumnRecords);
}
$(newDataSource).find("[id^=per-column-distribution-select]").selectpicker("val", dsCount.perColumnRecordsDistribution);
$(newDataSource).find("[id^=per-column-distribution-select]")[0].dispatchEvent(new Event("change"));
let updatedPerColumnDistribution = $(newDataSource).find("[id^=per-column-distribution-select]");
dispatchEvent(updatedPerColumnDistribution, "change");
if (dsCount.perColumnRecordsDistribution === "exponential") {
$(newDataSource).find("[id^=per-column-distribution-rate]").val(dsCount.perColumnRecordsDistributionRateParam);
}
Expand Down
15 changes: 10 additions & 5 deletions app/src/main/resources/ui/helper-validation.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ import {
createCloseButton,
createManualContainer,
createNewField,
findNextLevelNodesByClass
dispatchEvent,
findNextLevelNodesByClass,
} from "./shared.js";
import {validationTypeOptionsMap} from "./configuration-data.js";

Expand All @@ -25,7 +26,8 @@ export function incValidations() {
}

function createGroupByValidationFromPlan(newValidation, validationOpts, validation) {
$(newValidation).find("[aria-label=GroupByColumns]").val(validationOpts.groupByColumns)[0].dispatchEvent(new Event("input"));
let updatedGroupByCols = $(newValidation).find("[aria-label=GroupByColumns]").val(validationOpts.groupByColumns)
dispatchEvent(updatedGroupByCols, "input");
// can be nested validations

if (validation.nested && validation.nested.validations) {
Expand Down Expand Up @@ -69,16 +71,19 @@ async function createValidationsFromDataSource(dataSource, manualValidation) {
numValidations += 1;
let newValidation = await createNewField(numValidations, "validation");
$(manualValidation).children(".accordion").append(newValidation);
$(newValidation).find("select[class~=validation-type]").selectpicker("val", validation.type)[0].dispatchEvent(new Event("change"));
let updatedValidationType = $(newValidation).find("select[class~=validation-type]").selectpicker("val", validation.type);
dispatchEvent(updatedValidationType, "change");
let validationOpts = validation.options;
let mainContainer = $(newValidation).find("[id^=data-validation-container]")[0];

if (validation.type === "column" && validationOpts.column) {
$(newValidation).find("[aria-label=Field]").val(validationOpts.column)[0].dispatchEvent(new Event("input"));
let updatedValidationCol = $(newValidation).find("[aria-label=Field]").val(validationOpts.column)
dispatchEvent(updatedValidationCol, "input");
} else if (validation.type === "groupBy" && validationOpts.groupByColumns) {
createGroupByValidationFromPlan(newValidation, validationOpts, validation);
} else if (validation.type === "upstream" && validationOpts.upstreamTaskName) {
$(newValidation).find("[aria-label=UpstreamTaskName]").val(validationOpts.upstreamTaskName)[0].dispatchEvent(new Event("input"));
let updatedUpstreamTaskName = $(newValidation).find("[aria-label=UpstreamTaskName]").val(validationOpts.upstreamTaskName)
dispatchEvent(updatedUpstreamTaskName, "input");
// can be nested validations

if (validation.nested && validation.nested.validations) {
Expand Down
22 changes: 15 additions & 7 deletions app/src/main/resources/ui/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
createManualContainer,
createSelect,
createToast,
dispatchEvent,
executePlan,
getDataConnectionsAndAddToSelect,
manualContainerDetails,
Expand Down Expand Up @@ -203,16 +204,17 @@ async function createDataConnectionInput(index) {
dataConnectionCol.setAttribute("class", "col");
dataConnectionCol.append(dataConnectionSelect);

let iconDiv = createIconWithConnectionTooltip(dataConnectionSelect);
// provide opportunity to override non-connection options for metadata source (i.e. namespace, dataset)
let overrideOptionsContainer = document.createElement("div");
let iconDiv = createIconWithConnectionTooltip(dataConnectionSelect, overrideOptionsContainer, "data-source-property", index);
let iconCol = document.createElement("div");
iconCol.setAttribute("class", "col-md-auto");
iconCol.append(iconDiv);

// let inputGroup = createInputGroup(dataConnectionSelect, iconDiv, "col");
// $(inputGroup).find(".input-group").addClass("align-items-center");
baseTaskDiv.append(taskNameFormFloating, dataConnectionCol, iconCol);
let baseDivWithSelectOptions = await getDataConnectionsAndAddToSelect(dataConnectionSelect, baseTaskDiv, "dataSource");

return await getDataConnectionsAndAddToSelect(dataConnectionSelect, baseTaskDiv, "dataSource");
return [baseDivWithSelectOptions, overrideOptionsContainer];
}

/*
Expand All @@ -225,7 +227,7 @@ async function createDataSourceConfiguration(index, closeButton, divider) {
let divContainer = document.createElement("div");
divContainer.setAttribute("id", "data-source-config-container-" + index);
divContainer.setAttribute("class", "data-source-config-container");
let dataConnectionFormFloating = await createDataConnectionInput(index);
let [dataConnectionFormFloating, overrideConnectionOptionsContainer] = await createDataConnectionInput(index);
let dataConfigAccordion = document.createElement("div");
dataConfigAccordion.setAttribute("class", "accordion mt-2");
let dataGenConfigContainer = createDataConfigElement(index, "generation");
Expand All @@ -236,7 +238,7 @@ async function createDataSourceConfiguration(index, closeButton, divider) {
divContainer.append(divider);
}
dataConnectionFormFloating.append(closeButton);
divContainer.append(dataConnectionFormFloating, dataConfigAccordion);
divContainer.append(dataConnectionFormFloating, overrideConnectionOptionsContainer, dataConfigAccordion);
return divContainer;
}

Expand All @@ -255,6 +257,10 @@ function createReportConfiguration() {
}
}

function getOverrideConnectionOptions(dataSource, currentDataSource) {
currentDataSource["options"] = getOverrideConnectionOptionsAsMap(dataSource, currentDataSource);
}

createReportConfiguration();
submitForm();
savePlan();
Expand All @@ -274,6 +280,7 @@ function getPlanDetails(form) {
getGeneration(dataSource, currentDataSource);
getValidations(dataSource, currentDataSource);
getRecordCount(dataSource, currentDataSource);
getOverrideConnectionOptions(dataSource, currentDataSource);
allUserInputs.push(currentDataSource);
}

Expand Down Expand Up @@ -394,7 +401,8 @@ if (currUrlParams.includes("plan-name=")) {
let newDataSource = await createDataSourceForPlan(numDataSources);
tasksDetailsBody.append(newDataSource);
$(newDataSource).find(".task-name-field").val(dataSource.taskName);
$(newDataSource).find(".data-connection-name").selectpicker("val", dataSource.name)[0].dispatchEvent(new Event("change"));
let updatedConnectionName = $(newDataSource).find(".data-connection-name").selectpicker("val", dataSource.name);
dispatchEvent(updatedConnectionName, "change");

await createGenerationElements(dataSource, newDataSource, numDataSources);
createCountElementsFromPlan(dataSource, newDataSource);
Expand Down
Loading

0 comments on commit 83c3e5a

Please sign in to comment.