Add conditional installation for S3 integrations (#1518)

* Add workflows to integration format Signed-off-by: Simeon Widdis <[email protected]> * Render integration workflows on frontend Signed-off-by: Simeon Widdis <[email protected]> * Add ability to toggle workflows to frontend Signed-off-by: Simeon Widdis <[email protected]> * Add workflows to integration build options Signed-off-by: Simeon Widdis <[email protected]> * Add asset workflow filtering to builder Signed-off-by: Simeon Widdis <[email protected]> * Add enabled workflows to setup request Signed-off-by: Simeon Widdis <[email protected]> * Don't allow integration setup if no workflows enabled Signed-off-by: Simeon Widdis <[email protected]> * Add workflows to other integrations Signed-off-by: Simeon Widdis <[email protected]> * Improve header for workflows section Signed-off-by: Simeon Widdis <[email protected]> * Update snapshots Signed-off-by: Simeon Widdis <[email protected]> --------- Signed-off-by: Simeon Widdis <[email protected]> (cherry picked from commit 8874c8c) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Mar 13, 2024 · 66cfc55 · 66cfc55
1 parent 2d18a4f
commit 66cfc55
Show file tree

Hide file tree

Showing 16 changed files with 1,579 additions and 51 deletions.
diff --git a/...omponents/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/...omponents/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap
diff --git a/public/components/integrations/components/__tests__/setup_integration.test.tsx b/public/components/integrations/components/__tests__/setup_integration.test.tsx
@@ -53,4 +53,19 @@ describe('Integration Setup Page', () => {
       expect(wrapper).toMatchSnapshot();
     });
   });
+
+  it('Renders the S3 connector form without workflows', async () => {
+    const wrapper = mount(
+      <SetupIntegrationForm
+        config={{ ...TEST_INTEGRATION_SETUP_INPUTS, connectionType: 's3' }}
+        updateConfig={() => {}}
+        integration={{ ...TEST_INTEGRATION_CONFIG, workflows: undefined }}
+        setupCallout={{ show: false }}
+      />
+    );
+
+    await waitFor(() => {
+      expect(wrapper).toMatchSnapshot();
+    });
+  });
 });
diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts
@@ -282,7 +282,8 @@ export async function addIntegrationRequest(
   integration: IntegrationConfig,
   setToast: (title: string, color?: Color, text?: string | undefined) => void,
   name?: string,
-  dataSource?: string
+  dataSource?: string,
+  workflows?: string[]
 ): Promise<boolean> {
   const http = coreRefs.http!;
   if (addSample) {
@@ -298,7 +299,7 @@ export async function addIntegrationRequest(
 
   let response: boolean = await http
     .post(`${INTEGRATIONS_BASE}/store/${templateName}`, {
-      body: JSON.stringify({ name, dataSource }),
+      body: JSON.stringify({ name, dataSource, workflows }),
     })
     .then((res) => {
       setToast(`${name} integration successfully added!`, 'success');

diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx
@@ -8,6 +8,7 @@ import {
   EuiButton,
   EuiButtonEmpty,
   EuiCallOut,
+  EuiCheckableCard,
   EuiComboBox,
   EuiEmptyPrompt,
   EuiFieldText,
@@ -42,6 +43,7 @@ export interface IntegrationSetupInputs {
   connectionLocation: string;
   checkpointLocation: string;
   connectionTableName: string;
+  enabledWorkflows: string[];
 }
 
 type SetupCallout = { show: true; title: string; color?: Color; text?: string } | { show: false };
@@ -182,6 +184,38 @@ const runQuery = async (
   }
 };
 
+export function SetupWorkflowSelector({
+  integration,
+  useWorkflows,
+  toggleWorkflow,
+}: {
+  integration: IntegrationConfig;
+  useWorkflows: Map<string, boolean>;
+  toggleWorkflow: (name: string) => void;
+}) {
+  if (!integration.workflows) {
+    return null;
+  }
+
+  const cards = integration.workflows.map((workflow) => {
+    return (
+      <EuiCheckableCard
+        id={`workflow-checkbox-${workflow.name}`}
+        key={workflow.name}
+        label={workflow.label}
+        checkableType="checkbox"
+        value={workflow.name}
+        checked={useWorkflows.get(workflow.name)}
+        onChange={() => toggleWorkflow(workflow.name)}
+      >
+        {workflow.description}
+      </EuiCheckableCard>
+    );
+  });
+
+  return cards;
+}
+
 export function SetupIntegrationForm({
   config,
   updateConfig,
@@ -197,6 +231,25 @@ export function SetupIntegrationForm({
   const [isBucketBlurred, setIsBucketBlurred] = useState(false);
   const [isCheckpointBlurred, setIsCheckpointBlurred] = useState(false);
 
+  const [useWorkflows, setUseWorkflows] = useState(new Map<string, boolean>());
+  const toggleWorkflow = (name: string) => {
+    setUseWorkflows(new Map(useWorkflows.set(name, !useWorkflows.get(name))));
+  };
+
+  useEffect(() => {
+    if (integration.workflows) {
+      setUseWorkflows(new Map(integration.workflows.map((w) => [w.name, w.enabled_by_default])));
+    }
+  }, [integration.workflows]);
+
+  useEffect(() => {
+    updateConfig({
+      enabledWorkflows: [...useWorkflows.entries()].filter((w) => w[1]).map((w) => w[0]),
+    });
+    // If we add the updateConfig dep here, rendering crashes with "Maximum update depth exceeded"
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [useWorkflows]);
+
   useEffect(() => {
     const updateDataSources = async () => {
       const data = await suggestDataSources(config.connectionType);
@@ -339,12 +392,47 @@ export function SetupIntegrationForm({
               }}
             />
           </EuiFormRow>
+          {integration.workflows ? (
+            <>
+              <EuiSpacer />
+              <EuiText>
+                <h3>Installation Flows</h3>
+              </EuiText>
+              <EuiSpacer />
+              <EuiFormRow
+                label={'Flows'}
+                helpText={
+                  'Select from the available asset types based on your use case. Choose at least one.'
+                }
+                isInvalid={![...useWorkflows.values()].includes(true)}
+                error={['Must select at least one workflow.']}
+              >
+                <SetupWorkflowSelector
+                  integration={integration}
+                  useWorkflows={useWorkflows}
+                  toggleWorkflow={toggleWorkflow}
+                />
+              </EuiFormRow>
+            </>
+          ) : null}
         </>
       ) : null}
     </EuiForm>
   );
 }
 
+const prepareQuery = (query: string, config: IntegrationSetupInputs): string => {
+  let queryStr = query.replaceAll(
+    '{table_name}',
+    `${config.connectionDataSource}.default.${config.connectionTableName}`
+  );
+  queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation);
+  queryStr = queryStr.replaceAll('{s3_checkpoint_location}', config.checkpointLocation);
+  queryStr = queryStr.replaceAll('{object_name}', config.connectionTableName);
+  queryStr = queryStr.replaceAll(/\s+/g, ' ');
+  return queryStr;
+};
+
 const addIntegration = async ({
   config,
   integration,
@@ -375,22 +463,20 @@ const addIntegration = async ({
   } else if (config.connectionType === 's3') {
     const http = coreRefs.http!;
 
-    const assets = await http.get(`${INTEGRATIONS_BASE}/repository/${integration.name}/assets`);
+    const assets: { data: ParsedIntegrationAsset[] } = await http.get(
+      `${INTEGRATIONS_BASE}/repository/${integration.name}/assets`
+    );
 
-    // Queries must exist because we disable s3 if they're not present
     for (const query of assets.data.filter(
-      (a: ParsedIntegrationAsset): a is { type: 'query'; query: string; language: string } =>
+      (a: ParsedIntegrationAsset): a is ParsedIntegrationAsset & { type: 'query' } =>
         a.type === 'query'
     )) {
-      let queryStr = (query.query as string).replaceAll(
-        '{table_name}',
-        `${config.connectionDataSource}.default.${config.connectionTableName}`
-      );
+      // Skip any queries that have conditional workflows but aren't enabled
+      if (query.workflows && !query.workflows.some((w) => config.enabledWorkflows.includes(w))) {
+        continue;
+      }
 
-      queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation);
-      queryStr = queryStr.replaceAll('{s3_checkpoint_location}', config.checkpointLocation);
-      queryStr = queryStr.replaceAll('{object_name}', config.connectionTableName);
-      queryStr = queryStr.replaceAll(/\s+/g, ' ');
+      const queryStr = prepareQuery(query.query, config);
       const result = await runQuery(queryStr, config.connectionDataSource, sessionId);
       if (!result.ok) {
         setLoading(false);
@@ -400,15 +486,15 @@ const addIntegration = async ({
       sessionId = result.value.sessionId ?? sessionId;
     }
     // Once everything is ready, add the integration to the new datasource as usual
-    // TODO determine actual values here after more about queries is known
     const res = await addIntegrationRequest(
       false,
       integration.name,
       config.displayName,
       integration,
       setCalloutLikeToast,
       config.displayName,
-      `flint_${config.connectionDataSource}_default_${config.connectionTableName}_mview`
+      `flint_${config.connectionDataSource}_default_${config.connectionTableName}_mview`,
+      config.enabledWorkflows
     );
     if (!res) {
       setLoading(false);
@@ -418,11 +504,14 @@ const addIntegration = async ({
   }
 };
 
-const isConfigValid = (config: IntegrationSetupInputs): boolean => {
+const isConfigValid = (config: IntegrationSetupInputs, integration: IntegrationConfig): boolean => {
   if (config.displayName.length < 1 || config.connectionDataSource.length < 1) {
     return false;
   }
   if (config.connectionType === 's3') {
+    if (integration.workflows && config.enabledWorkflows.length < 1) {
+      return false;
+    }
     return (
       config.connectionLocation.startsWith('s3://') && config.checkpointLocation.startsWith('s3://')
     );
@@ -477,7 +566,7 @@ export function SetupBottomBar({
             iconType="arrowRight"
             iconSide="right"
             isLoading={loading}
-            disabled={!isConfigValid(config)}
+            disabled={!isConfigValid(config, integration)}
             onClick={async () =>
               addIntegration({ integration, config, setLoading, setCalloutLikeToast })
             }
@@ -511,6 +600,7 @@ export function SetupIntegrationPage({ integration }: { integration: string }) {
     connectionLocation: '',
     checkpointLocation: '',
     connectionTableName: integration,
+    enabledWorkflows: [],
   });
 
   const [template, setTemplate] = useState({

diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json
@@ -8,6 +8,20 @@
   "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"],
   "author": "OpenSearch",
   "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info",
+  "workflows": [
+    {
+      "name": "queries",
+      "label": "Queries (recommended)",
+      "description": "Tables and pre-written queries for quickly getting insights on your data.",
+      "enabled_by_default": true
+    },
+    {
+      "name": "dashboards",
+      "label": "Dashboards & Visualizations",
+      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+      "enabled_by_default": false
+    }
+  ],
   "statics": {
     "logo": {
       "annotation": "ELB Logo",
@@ -51,7 +65,8 @@
       "name": "aws_elb",
       "version": "1.0.0",
       "extension": "ndjson",
-      "type": "savedObjectBundle"
+      "type": "savedObjectBundle",
+      "workflows": ["dashboards"]
     },
     {
       "name": "create_table",
@@ -63,7 +78,8 @@
       "name": "create_mv",
       "version": "1.0.0",
       "extension": "sql",
-      "type": "query"
+      "type": "query",
+      "workflows": ["dashboards"]
     }
   ],
   "sampleData": {

diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json
@@ -8,6 +8,20 @@
   "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
   "author": "Haidong Wang",
   "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info",
+  "workflows": [
+    {
+      "name": "queries",
+      "label": "Queries (recommended)",
+      "description": "Tables and pre-written queries for quickly getting insights on your data.",
+      "enabled_by_default": true
+    },
+    {
+      "name": "dashboards",
+      "label": "Dashboards & Visualizations",
+      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+      "enabled_by_default": false
+    }
+  ],
   "statics": {
     "logo": {
       "annotation": "AWS VPC Logo",
@@ -47,7 +61,8 @@
       "name": "aws_vpc_flow",
       "version": "1.0.0",
       "extension": "ndjson",
-      "type": "savedObjectBundle"
+      "type": "savedObjectBundle",
+      "workflows": ["dashboards"]
     },
     {
       "name": "create_table_vpc",
@@ -59,7 +74,8 @@
       "name": "create_mv_vpc",
       "version": "1.0.0",
       "extension": "sql",
-      "type": "query"
+      "type": "query",
+      "workflows": ["dashboards"]
     }
   ],
   "sampleData": {

diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json
@@ -8,6 +8,20 @@
   "labels": ["Observability", "Logs", "Flint S3"],
   "author": "OpenSearch",
   "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
+  "workflows": [
+    {
+      "name": "queries",
+      "label": "Queries (recommended)",
+      "description": "Tables and pre-written queries for quickly getting insights on your data.",
+      "enabled_by_default": true
+    },
+    {
+      "name": "dashboards",
+      "label": "Dashboards & Visualizations",
+      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+      "enabled_by_default": false
+    }
+  ],
   "statics": {
     "logo": {
       "annotation": "NginX Logo",
@@ -43,7 +57,8 @@
       "name": "nginx",
       "version": "1.0.0",
       "extension": "ndjson",
-      "type": "savedObjectBundle"
+      "type": "savedObjectBundle",
+      "workflows": ["dashboards"]
     },
     {
       "name": "create_table",
@@ -55,7 +70,8 @@
       "name": "create_mv",
       "version": "1.0.0",
       "extension": "sql",
-      "type": "query"
+      "type": "query",
+      "workflows": ["dashboards"]
     }
   ],
   "sampleData": {