Web console basic end-to-end-test (apache#9595)

Load data and query (i.e., automate
https://druid.apache.org/docs/latest/tutorials/tutorial-batch.html) to
have some basic checks ensuring the web console is wired up to druid
correctly.

The new end-to-end tests (tutorial-batch.spec.ts) are added to
`web-console/e2e-tests`. Within that directory:
- `components` represent the various tabs of the web console. Currently,
  abstractions for `load data`, `ingestion`, `datasources`, and `query`
  are implemented.
- `components/load-data/data-connector` contains abstractions for the
  different data source options available to the data loader's `Connect`
  step. Currently, only the `Local file` data source connector is
  implemented.
- `components/load-data/config` contains abstractions for the different
  configuration options available for each step of the data loader flow.
  Currently, the `Configure Schema`, `Partition`, and `Publish` steps
  have initial implementation of their configuration options.
- `util` contains various helper methods for the tests and does not
  contain abstractions of the web console.

Changes to add the new tests to CI:
- `.travis.yml`: New "web console end-to-end tests" job
- `web-console/jest.*.js`: Refactor jest configurations to have
  different flavors for unit tests and for end-to-end tests. In
  particular, the latter adds a jest setup configuration to wait for the
  web console to be ready (`web-console/e2e-tests/util/setup.ts`).
- `web-console/package.json`: Refactor run scripts to add new script for
  running end-to-end tests.
- `web-console/script/druid`: Utility scripts for building, starting,
  and stopping druid.

Other changes:
- `pom.xml`: Refactor various settings disable java static checks and to
  disable java tests into two new maven profiles. Since the same
  settings are used in several places (e.g., .travis.yml, Dockerfiles,
  etc.), having them in maven profiles makes it more maintainable.
- `web-console/src/console-application.tsx`: Fix typo ("the the").

.gitignore

@@ -11,6 +11,7 @@ target
 .project
 .PVS-Studio
 .settings/
+.vscode
 *.log
 *.DS_Store
 _site

.travis.yml

@@ -30,17 +30,8 @@ env:
     - DOCKER_IP=127.0.0.1  # for integration tests
     - MVN="mvn -B"
     - > # Various options to make execution of maven goals faster (e.g., mvn install)
-      MAVEN_SKIP="
-      -Danimal.sniffer.skip=true
-      -Dcheckstyle.skip=true
-      -Ddruid.console.skip=true
-      -Denforcer.skip=true
-      -Dforbiddenapis.skip=true
-      -Dmaven.javadoc.skip=true
-      -Dpmd.skip=true
-      -Dspotbugs.skip=true
-      "
-    - MAVEN_SKIP_TESTS="-DskipTests -Djacoco.skip=true"
+      MAVEN_SKIP="-Pskip-static-checks -Ddruid.console.skip=true -Dmaven.javadoc.skip=true"
+    - MAVEN_SKIP_TESTS="-Pskip-tests"
 
 # Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not
 # needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any
@@ -254,14 +245,20 @@ jobs:
       name: "(openjdk11) other modules test (SQL Compatibility)"
       jdk: openjdk11
 
-    - &test_webconsole
-      name: "web console"
+    - name: "web console"
       install: skip
       script:
         - ${MVN} test -pl 'web-console'
       after_success:
         - (cd web-console && travis_retry npm run codecov)  # retry in case of network error
 
+    - name: "web console end-to-end test"
+      before_install: *setup_generate_license
+      install: web-console/script/druid build
+      before_script: web-console/script/druid start
+      script: (cd web-console && npm run test-e2e)
+      after_script:  web-console/script/druid stop
+
     - name: "docs"
       install: (cd website && npm install)
       script: |-

distribution/docker/Dockerfile

@@ -28,14 +28,8 @@ WORKDIR /src
 RUN mvn -B -ff -q dependency:go-offline \
       install \
       -Pdist,bundle-contrib-exts \
-      -DskipTests \
-      -Danimal.sniffer.skip=true \
-      -Dcheckstyle.skip=true \
-      -Denforcer.skip=true \
-      -Dforbiddenapis.skip=true \
-      -Dmaven.javadoc.skip=true \
-      -Dpmd.skip=true \
-      -Dspotbugs.skip=true
+      -Pskip-static-checks,skip-tests \
+      -Dmaven.javadoc.skip=true
 
 RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \
       -Dexpression=project.version -DforceStdout=true \

distribution/docker/Dockerfile.java11

@@ -28,14 +28,8 @@ WORKDIR /src
 RUN mvn -B -ff -q dependency:go-offline \
       install \
       -Pdist,bundle-contrib-exts \
-      -DskipTests \
-      -Danimal.sniffer.skip=true \
-      -Dcheckstyle.skip=true \
-      -Denforcer.skip=true \
-      -Dforbiddenapis.skip=true \
-      -Dmaven.javadoc.skip=true \
-      -Dpmd.skip=true \
-      -Dspotbugs.skip=true
+      -Pskip-static-checks,skip-tests \
+      -Dmaven.javadoc.skip=true
 
 RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \
       -Dexpression=project.version -DforceStdout=true \

pom.xml

@@ -1947,5 +1947,24 @@
             <module>website</module>
           </modules>
         </profile>
+        <profile>
+          <id>skip-static-checks</id>
+          <properties>
+            <animal.sniffer.skip>true</animal.sniffer.skip>
+            <checkstyle.skip>true</checkstyle.skip>
+            <enforcer.skip>true</enforcer.skip>
+            <forbiddenapis.skip>true</forbiddenapis.skip>
+            <maven.javadoc.skip>true</maven.javadoc.skip>
+            <pmd.skip>true</pmd.skip>
+            <spotbugs.skip>true</spotbugs.skip>
+          </properties>
+        </profile>
+        <profile>
+          <id>skip-tests</id>
+          <properties>
+            <skipTests>true</skipTests>
+            <jacoco.skip>true</jacoco.skip>
+          </properties>
+        </profile>
     </profiles>
 </project>

web-console/e2e-tests/component/datasources/datasource.ts

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Represents datasource row in datasource overview table.
+ */
+export class Datasource {
+  constructor(props: DatasourceProps) {
+    Object.assign(this, props);
+  }
+}
+
+interface DatasourceProps {
+  readonly name: string;
+  readonly availability: string;
+  readonly numRows: number;
+}
+
+export interface Datasource extends DatasourceProps {}

web-console/e2e-tests/component/datasources/overview.ts

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as playwright from 'playwright-core';
+
+import { extractTable } from '../../util/table';
+
+import { Datasource } from './datasource';
+
+/**
+ * Datasource overview table column identifiers.
+ */
+enum DatasourceColumn {
+  NAME = 0,
+  AVAILABILITY,
+  SEGMENT_LOAD_DROP,
+  RETENTION,
+  REPLICATED_SIZE,
+  SIZE,
+  COMPACTION,
+  AVG_SEGMENT_SIZE,
+  NUM_ROWS,
+}
+
+/**
+ * Represents datasource overview tab.
+ */
+export class DatasourcesOverview {
+  private readonly page: playwright.Page;
+  private readonly baseUrl: string;
+
+  constructor(page: playwright.Page, unifiedConsoleUrl: string) {
+    this.page = page;
+    this.baseUrl = unifiedConsoleUrl + '#datasources';
+  }
+
+  async getDatasources(): Promise<Datasource[]> {
+    await this.page.goto(this.baseUrl);
+    await this.page.reload({ waitUntil: 'networkidle0' });
+
+    const data = await extractTable(this.page, 'div div.rt-tr-group', 'div.rt-td');
+
+    return data.map(
+      row =>
+        new Datasource({
+          name: row[DatasourceColumn.NAME],
+          availability: row[DatasourceColumn.AVAILABILITY],
+          numRows: DatasourcesOverview.parseNumber(row[DatasourceColumn.NUM_ROWS]),
+        }),
+    );
+  }
+
+  private static parseNumber(text: string): number {
+    return Number(text.replace(/,/g, ''));
+  }
+}

web-console/e2e-tests/component/ingestion/overview.ts

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as playwright from 'playwright-core';
+
+import { extractTable } from '../../util/table';
+
+import { IngestionTask } from './task';
+
+/**
+ * Ingestion overview task table column identifiers.
+ */
+enum TaskColumn {
+  TASK_ID = 0,
+  GROUP_ID,
+  TYPE,
+  DATASOURCE,
+  LOCATION,
+  CREATED_TIME,
+  STATUS,
+  DURATION,
+}
+
+/**
+ * Represents ingestion overview tab.
+ */
+export class IngestionOverview {
+  private readonly page: playwright.Page;
+  private readonly baseUrl: string;
+
+  constructor(page: playwright.Page, unifiedConsoleUrl: string) {
+    this.page = page;
+    this.baseUrl = unifiedConsoleUrl + '#ingestion';
+  }
+
+  async getTasks(): Promise<IngestionTask[]> {
+    await this.page.goto(this.baseUrl);
+    await this.page.reload({ waitUntil: 'networkidle0' });
+
+    const data = await extractTable(this.page, 'div.bottom-pane div.rt-tr-group', 'div.rt-td');
+
+    return data.map(
+      row =>
+        new IngestionTask({
+          datasource: row[TaskColumn.DATASOURCE],
+          status: row[TaskColumn.STATUS],
+        }),
+    );
+  }
+}

web-console/e2e-tests/component/ingestion/task.ts

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Represents row in ingestion overview task table.
+ */
+export class IngestionTask {
+  constructor(props: IngestionTaskProps) {
+    Object.assign(this, props);
+  }
+}
+
+interface IngestionTaskProps {
+  readonly datasource: string;
+  readonly status: string;
+}
+
+export interface IngestionTask extends IngestionTaskProps {}

web-console/e2e-tests/component/load-data/config/configure-schema.ts

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Data loader configure schema step configuration.
+ */
+export class ConfigureSchemaConfig {
+  constructor(props: ConfigureSchemaConfigProps) {
+    Object.assign(this, props);
+  }
+}
+
+interface ConfigureSchemaConfigProps {
+  readonly rollup: boolean;
+}
+
+export interface ConfigureSchemaConfig extends ConfigureSchemaConfigProps {}

web-console/e2e-tests/component/load-data/config/partition.ts

@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Possible values for partition step segment granularity.
+ */
+export enum SegmentGranularity {
+  HOUR = 'HOUR',
+  DAY = 'DAY',
+  MONTH = 'MONTH',
+  YEAR = 'YEAR',
+}
+
+/**
+ * Data loader partition step configuration.
+ */
+export class PartitionConfig {
+  constructor(props: PartitionConfigProps) {
+    Object.assign(this, props);
+  }
+}
+
+interface PartitionConfigProps {
+  readonly segmentGranularity: SegmentGranularity;
+}
+
+export interface PartitionConfig extends PartitionConfigProps {}