Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move create acceleration flyout from workbench to datasources #1508

Merged
merged 2 commits into from
Mar 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions common/constants/data_sources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,53 @@ export enum DATA_SOURCE_TYPES {
export const ASYNC_POLLING_INTERVAL = 2000;

export const CATALOG_CACHE_VERSION = '1.0';
export const ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME = 'skipping';
export const ACCELERATION_TIME_INTERVAL = [
{ text: 'millisecond(s)', value: 'millisecond' },
{ text: 'second(s)', value: 'second' },
{ text: 'minutes(s)', value: 'minute' },
{ text: 'hour(s)', value: 'hour' },
{ text: 'day(s)', value: 'day' },
{ text: 'week(s)', value: 'week' },
];

export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)';
export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z][a-z_]*$/;
export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/;

export const ACCELERATION_INDEX_TYPES = [
{ label: 'Skipping Index', value: 'skipping' },
{ label: 'Covering Index', value: 'covering' },
{ label: 'Materialized View', value: 'materialized' },
];

export const ACC_INDEX_TYPE_DOCUMENTATION_URL =
'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md';

export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix_<index name>_suffix\`. They share a common prefix structure, which is \`flint_<data source name>_<database name>_<table name>_\`. Additionally, they may have a suffix that varies based on the index type.
##### Skipping Index
- For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`.
- An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`.
##### Covering Index
- 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`.
- For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`.
##### Materialized View Index
- 'Materialized View' indices also enable users to define their index name, but they do not have a suffix.
- An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`.
##### Note:
- All user given index names must be in lowercase letters. Index name cannot begin with underscores. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed.
`;

export const SKIPPING_INDEX_ACCELERATION_METHODS = [
{ value: 'PARTITION', text: 'Partition' },
{ value: 'VALUE_SET', text: 'Value Set' },
{ value: 'MIN_MAX', text: 'Min Max' },
];

export const ACCELERATION_AGGREGRATION_FUNCTIONS = [
{ label: 'count' },
{ label: 'sum' },
{ label: 'avg' },
{ label: 'max' },
{ label: 'min' },
];
100 changes: 100 additions & 0 deletions common/types/data_connections.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
interface AsyncApiDataResponse {
status: string;
schema?: Array<{ name: string; type: string }>;
datarows?: any;

Check warning on line 54 in common/types/data_connections.ts

View workflow job for this annotation

GitHub Actions / Lint

Unexpected any. Specify a different type
total?: number;
size?: number;
error?: string;
Expand Down Expand Up @@ -128,3 +128,103 @@
}

export type AsyncPollingResult = PollingSuccessResult | null;

export interface CreateAccelerationForm {
dataSource: string;
database: string;
dataTable: string;
dataTableFields: DataTableFieldsType[];
accelerationIndexType: AccelerationIndexType;
skippingIndexQueryData: SkippingIndexRowType[];
coveringIndexQueryData: string[];
materializedViewQueryData: MaterializedViewQueryType;
accelerationIndexName: string;
primaryShardsCount: number;
replicaShardsCount: number;
refreshType: AccelerationRefreshType;
checkpointLocation: string | undefined;
watermarkDelay: WatermarkDelayType;
refreshIntervalOptions: RefreshIntervalType;
formErrors: FormErrorsType;
}

export type AggregationFunctionType = 'count' | 'sum' | 'avg' | 'max' | 'min';

export interface MaterializedViewColumn {
id: string;
functionName: AggregationFunctionType;
functionParam: string;
fieldAlias?: string;
}

export type SkippingIndexAccMethodType = 'PARTITION' | 'VALUE_SET' | 'MIN_MAX';

export interface SkippingIndexRowType {
id: string;
fieldName: string;
dataType: string;
accelerationMethod: SkippingIndexAccMethodType;
}

export interface DataTableFieldsType {
id: string;
fieldName: string;
dataType: string;
}

export interface RefreshIntervalType {
refreshWindow: number;
refreshInterval: string;
}

export interface WatermarkDelayType {
delayWindow: number;
delayInterval: string;
}

export interface GroupByTumbleType {
timeField: string;
tumbleWindow: number;
tumbleInterval: string;
}

export interface MaterializedViewQueryType {
columnsValues: MaterializedViewColumn[];
groupByTumbleValue: GroupByTumbleType;
}

export interface FormErrorsType {
dataSourceError: string[];
databaseError: string[];
dataTableError: string[];
skippingIndexError: string[];
coveringIndexError: string[];
materializedViewError: string[];
indexNameError: string[];
primaryShardsError: string[];
replicaShardsError: string[];
refreshIntervalError: string[];
checkpointLocationError: string[];
watermarkDelayError: string[];
}

export type AccelerationRefreshType = 'auto' | 'interval' | 'manual';

export interface CreateAccelerationForm {
dataSource: string;
database: string;
dataTable: string;
dataTableFields: DataTableFieldsType[];
accelerationIndexType: AccelerationIndexType;
skippingIndexQueryData: SkippingIndexRowType[];
coveringIndexQueryData: string[];
materializedViewQueryData: MaterializedViewQueryType;
accelerationIndexName: string;
primaryShardsCount: number;
replicaShardsCount: number;
refreshType: AccelerationRefreshType;
checkpointLocation: string | undefined;
watermarkDelay: WatermarkDelayType;
refreshIntervalOptions: RefreshIntervalType;
formErrors: FormErrorsType;
}
Loading
Loading