Skip to content

Commit

Permalink
feat(#3282): Add a regex rule for string event properties to connect
Browse files Browse the repository at this point in the history
  • Loading branch information
tenthe committed Oct 7, 2024
1 parent ffd5e7c commit bbc0a69
Show file tree
Hide file tree
Showing 22 changed files with 383 additions and 24 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription;
import org.apache.streampipes.model.schema.EventProperty;
Expand Down Expand Up @@ -80,6 +81,13 @@ public void visit(RenameRuleDescription rule) {
}
}

@Override
public void visit(RegexTransformationRuleDescription rule) {
if (containsKey(rule.getRuntimeKey())) {
validRules.add(rule);
}
}

@Override
public void visit(EventRateTransformationRuleDescription rule) {
// Do nothing
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription;
import org.apache.streampipes.model.shared.annotation.TsModel;
Expand All @@ -47,6 +48,7 @@
@JsonSubTypes.Type(CreateNestedRuleDescription.class),
@JsonSubTypes.Type(DeleteRuleDescription.class),
@JsonSubTypes.Type(RenameRuleDescription.class),
@JsonSubTypes.Type(RegexTransformationRuleDescription.class),
@JsonSubTypes.Type(MoveRuleDescription.class),
@JsonSubTypes.Type(ChangeDatatypeTransformationRuleDescription.class),
@JsonSubTypes.Type(CorrectionValueTransformationRuleDescription.class),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription;
import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription;
import org.apache.streampipes.model.grounding.TopicDefinition;
Expand Down Expand Up @@ -94,6 +95,8 @@ public static GsonBuilder getGsonBuilder() {

builder.registerTypeAdapterFactory(RuntimeTypeAdapterFactory.of(TransformationRuleDescription.class, "sourceType")
.registerSubtype(RenameRuleDescription.class, "org.apache.streampipes.model.RenameRuleDescription")
.registerSubtype(RegexTransformationRuleDescription.class,
"org.apache.streampipes.model.RegexTransformationRuleDescription")
.registerSubtype(MoveRuleDescription.class, "org.apache.streampipes.model.MoveRuleDescription")
.registerSubtype(DeleteRuleDescription.class, "org.apache.streampipes.model.DeleteRuleDescription")
.registerSubtype(CreateNestedRuleDescription.class, "org.apache.streampipes.model.CreateNestedRuleDescription")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
density;sensorId
122.0;aensor01
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
density;sensorId
122.0;aenaor01
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
density;sensorId
122.0;enor01
2 changes: 2 additions & 0 deletions ui/cypress/fixtures/connect/regexRule/input.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
timestamp;density;sensorId
1720018277000;122.0;sensor01
17 changes: 16 additions & 1 deletion ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,10 @@ export class ConnectEventSchemaUtils {
cy.get('#event-schema-next-button').parent().should('not.be.disabled');
}

public static schemaPreviewResultEvent() {
return cy.dataCy('schema-preview-result-event', { timeout: 10000 });
}

public static finishEventSchemaConfiguration() {
// Click next
cy.dataCy('sp-connect-schema-editor', { timeout: 10000 }).should(
Expand All @@ -270,7 +274,18 @@ export class ConnectEventSchemaUtils {
ConnectEventSchemaUtils.validateRuntimeName(propertyName);
}

//
public static regexValueInput() {
return cy.dataCy('regex-value');
}

public static regexReplaceWithValueInput() {
return cy.dataCy('regex-replace-with-value');
}

public static regexReplaceAllCheckbox() {
return cy.dataCy('regex-replace-all-value');
}

/**
* Function to escape special characters in a string for use in Cypress
* selectors
Expand Down
12 changes: 6 additions & 6 deletions ui/cypress/tests/connect/rules/deleteTransformationRule.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ describe('Connect delete rule transformation', () => {
ConnectEventSchemaUtils.deleteProperty('reserved bit_1');
ConnectEventSchemaUtils.deleteProperty('reserved bit_2');

cy.dataCy('schema-preview-result-event').should(
ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',
'{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01715356080000}',
);
Expand Down Expand Up @@ -69,7 +69,7 @@ describe('Connect delete rule transformation', () => {
ConnectEventSchemaUtils.deleteProperty('child');

// The resulting string contains non-breaking spaces character (\u00A0)
cy.dataCy('schema-preview-result-event').should(
ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',
'{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{\u00A0\u00A0\u00A0\u00A0\u00A0\u00A0\u00A0\u00A0"child_two":\u00A0"textTwo"\u00A0\u00A0\u00A0\u00A0},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
);
Expand All @@ -78,7 +78,7 @@ describe('Connect delete rule transformation', () => {

// Test to delete the parent property
ConnectEventSchemaUtils.deleteProperty('parent');
cy.dataCy('schema-preview-result-event').should(
ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',

'{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
Expand All @@ -88,7 +88,7 @@ describe('Connect delete rule transformation', () => {
ConnectEventSchemaUtils.deleteProperty('child');
ConnectEventSchemaUtils.deleteProperty('child_two');

cy.dataCy('schema-preview-result-event').should(
ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',
'{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
);
Expand All @@ -97,7 +97,7 @@ describe('Connect delete rule transformation', () => {

// Test to delete the parent property
ConnectEventSchemaUtils.deleteProperty('parent');
cy.dataCy('schema-preview-result-event').should(
ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',

'{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
Expand All @@ -109,7 +109,7 @@ describe('Connect delete rule transformation', () => {
ConnectEventSchemaUtils.deleteProperty('child');
ConnectEventSchemaUtils.deleteProperty('child_two');

cy.dataCy('schema-preview-result-event').should(
ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',
'{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
);
Expand Down
107 changes: 107 additions & 0 deletions ui/cypress/tests/connect/rules/regexRule.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
import { ConnectEventSchemaUtils } from '../../../support/utils/connect/ConnectEventSchemaUtils';
import { ConnectBtns } from '../../../support/utils/connect/ConnectBtns';
import { FileManagementUtils } from '../../../support/utils/FileManagementUtils';

describe('Connect value rule transformations', () => {
beforeEach('Setup Test', () => {
cy.initStreamPipesTest();
FileManagementUtils.addFile('connect/regexRule/input.csv');
});

it('Perform Test Replace All True', () => {
runTest('aenaor01', 'a', true, 'expectedReplaceAllTrue.csv');
});

it('Perform Test Replace All False', () => {
runTest('aensor01', 'a', false, 'expectedReplaceAllFalse.csv');
});

it('Perform Test Replace With Empty String', () => {
runTest(
'enor01',
'',
true,
'expectedReplaceAllTrueWithEmptyString.csv',
);
});

function runTest(
expectedPreview: string,
replaceWith: string,
replaceAll: boolean,
expecedCsvFile: string,
) {
const adapterConfiguration =
ConnectUtils.setUpPreprocessingRuleTest(false);

// Ensure regex input is not shown for number property
ConnectEventSchemaUtils.clickEditProperty('density');
ConnectEventSchemaUtils.regexValueInput().should('not.exist');
ConnectBtns.saveEditProperty().click();
ConnectEventSchemaUtils.clickEditProperty('sensorId');
ConnectEventSchemaUtils.regexValueInput().should('be.visible');

// Validate that preview works
ConnectEventSchemaUtils.regexValueInput().type('s');
if (replaceWith !== '') {
ConnectEventSchemaUtils.regexReplaceWithValueInput().type(
replaceWith,
);
}
if (replaceAll) {
ConnectEventSchemaUtils.regexReplaceAllCheckbox().click();
}

ConnectBtns.saveEditProperty().click();

ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'contain.text',
expectedPreview,
);
ConnectEventSchemaUtils.markPropertyAsTimestamp('timestamp');
ConnectEventSchemaUtils.finishEventSchemaConfiguration();

ConnectUtils.tearDownPreprocessingRuleTest(
adapterConfiguration,
'cypress/fixtures/connect/regexRule/' + expecedCsvFile,
true,
);
}
});
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/

/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
// Generated using typescript-generator version 3.2.1263 on 2024-09-08 22:18:46.
// Generated using typescript-generator version 3.2.1263 on 2024-10-07 13:58:29.

import { Storable } from './streampipes-model';
import { Storable } from '@streampipes/platform-services';

export class Group implements Storable {
elementId: string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/

/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
// Generated using typescript-generator version 3.2.1263 on 2024-09-30 14:11:40.
// Generated using typescript-generator version 3.2.1263 on 2024-10-07 13:58:27.

export class NamedStreamPipesEntity implements Storable {
'@class':
Expand Down Expand Up @@ -209,6 +208,7 @@ export class TransformationRuleDescription {
| 'org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription'
| 'org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription'
| 'org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription'
| 'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription'
| 'org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription'
| 'org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription'
| 'org.apache.streampipes.model.connect.rules.stream.StreamTransformationRuleDescription'
Expand Down Expand Up @@ -261,6 +261,8 @@ export class TransformationRuleDescription {
return DeleteRuleDescription.fromData(data);
case 'org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription':
return RenameRuleDescription.fromData(data);
case 'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription':
return RegexTransformationRuleDescription.fromData(data);
case 'org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription':
return MoveRuleDescription.fromData(data);
case 'org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription':
Expand All @@ -282,6 +284,7 @@ export class ValueTransformationRuleDescription extends TransformationRuleDescri
| 'org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription'
| 'org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription'
| 'org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription'
| 'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription'
| 'org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription'
| 'org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription';

Expand Down Expand Up @@ -316,6 +319,8 @@ export class ValueTransformationRuleDescription extends TransformationRuleDescri
return CorrectionValueTransformationRuleDescription.fromData(
data,
);
case 'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription':
return RegexTransformationRuleDescription.fromData(data);
}
}
}
Expand Down Expand Up @@ -3061,6 +3066,30 @@ export class QuantitativeValue extends ValueSpecification {
}
}

export class RegexTransformationRuleDescription extends ValueTransformationRuleDescription {
'@class': 'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription';
'regex': string;
'replaceAll': boolean;
'replaceWith': string;
'runtimeKey': string;

static 'fromData'(
data: RegexTransformationRuleDescription,
target?: RegexTransformationRuleDescription,
): RegexTransformationRuleDescription {
if (!data) {
return data;
}
const instance = target || new RegexTransformationRuleDescription();
super.fromData(data, instance);
instance.regex = data.regex;
instance.replaceAll = data.replaceAll;
instance.replaceWith = data.replaceWith;
instance.runtimeKey = data.runtimeKey;
return instance;
}
}

export class RemoveDuplicatesTransformationRuleDescription extends StreamTransformationRuleDescription {
'@class': 'org.apache.streampipes.model.connect.rules.stream.RemoveDuplicatesTransformationRuleDescription';
'filterTimeWindow': string;
Expand Down Expand Up @@ -4080,6 +4109,7 @@ export type TransformationRuleDescriptionUnion =
| CreateNestedRuleDescription
| DeleteRuleDescription
| RenameRuleDescription
| RegexTransformationRuleDescription
| MoveRuleDescription
| ChangeDatatypeTransformationRuleDescription
| CorrectionValueTransformationRuleDescription;
Expand All @@ -4100,7 +4130,8 @@ export type ValueTransformationRuleDescriptionUnion =
| AddValueTransformationRuleDescription
| TimestampTranfsformationRuleDescription
| UnitTransformRuleDescription
| CorrectionValueTransformationRuleDescription;
| CorrectionValueTransformationRuleDescription
| RegexTransformationRuleDescription;

function __getCopyArrayFn<T>(itemCopyFn: (item: T) => T): (array: T[]) => T[] {
return (array: T[]) => __copyArray(array, itemCopyFn);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,8 @@ export class DataType {
public static isBooleanType(datatype: string): boolean {
return datatype === DataType.BOOLEAN;
}

public static isStringType(datatype: string): boolean {
return datatype === DataType.STRING;
}
}
2 changes: 2 additions & 0 deletions ui/src/app/connect/connect.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ import { AdapterStatusLightComponent } from './components/existing-adapters/adap
import { SpAdapterDeploymentSettingsComponent } from './components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component';
import { SpAdapterDocumentationDialogComponent } from './dialog/adapter-documentation/adapter-documentation-dialog.component';
import { AdapterDetailsDataComponent } from './components/adapter-details/adapter-details-data/adapter-details-data.component';
import { EditRegexTransformationComponent } from './dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component';

@NgModule({
imports: [
Expand Down Expand Up @@ -204,6 +205,7 @@ import { AdapterDetailsDataComponent } from './components/adapter-details/adapte
DeleteAdapterDialogComponent,
EventSchemaComponent,
EditEventPropertyComponent,
EditRegexTransformationComponent,
EventPropertyRowComponent,
EditUnitTransformationComponent,
EditSchemaTransformationComponent,
Expand Down
Loading

0 comments on commit bbc0a69

Please sign in to comment.