Skip to content

Commit

Permalink
Merge branch 'main' into use-entity-name
Browse files Browse the repository at this point in the history
  • Loading branch information
ayush-shah authored Jan 15, 2025
2 parents 0e5ae15 + cbbef9e commit 83a6e39
Show file tree
Hide file tree
Showing 2,438 changed files with 40,955 additions and 14,533 deletions.
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Individual scopes

# Review from UI owners for changes around UI code
/openmetadata-ui/src/main/resources/ui/ @ShaileshParmar11 @karanh37 @chirag-madlani @Sachin-chaurasiya
/openmetadata-ui/src/main/resources/ui/ @open-metadata/ui

# Review from Backend owners for changes around Backend code
/openmetadata-service/ @open-metadata/backend
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/auto-cherry-pick-labeled-prs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@ run-name: OpenMetadata release cherry-pick PR #${{ github.event.pull_request.num

# yamllint disable-line rule:truthy
on:
pull_request:
pull_request_target:
types: [closed]
branches:
- main
permissions:
contents: write
pull-requests: write
issues: write
env:
CURRENT_RELEASE_ENDPOINT: ${{ vars.CURRENT_RELEASE_ENDPOINT }} # Endpoint that returns the current release version in json format
jobs:
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/playwright-mysql-e2e-skip.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ jobs:
playwright-ci-mysql:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
shardIndex: [1, 2]
shardTotal: [2]
fail-fast: false
matrix:
shardIndex: [1, 2]
shardTotal: [2]
steps:
- run: 'echo "Step is not required"'
2 changes: 1 addition & 1 deletion .github/workflows/playwright-mysql-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ concurrency:

jobs:
playwright-ci-mysql:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
environment: test
strategy:
fail-fast: false
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/playwright-postgresql-e2e-skip.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ jobs:
playwright-ci-postgresql:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
shardIndex: [1, 2]
shardTotal: [2]
fail-fast: false
matrix:
shardIndex: [1, 2]
shardTotal: [2]
steps:
- run: 'echo "Step is not required"'
2 changes: 1 addition & 1 deletion .github/workflows/playwright-postgresql-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ concurrency:

jobs:
playwright-ci-postgresql:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
environment: test
strategy:
fail-fast: false
Expand Down
Empty file.
44 changes: 44 additions & 0 deletions bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
-- add timestamp index for test case result reindex performance
ALTER TABLE data_quality_data_time_series ADD INDEX `idx_timestamp_desc` (timestamp DESC);

CREATE TABLE background_jobs (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
jobType VARCHAR(256) NOT NULL,
methodName VARCHAR(256) NOT NULL,
jobArgs JSON NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'PENDING',
createdBy VARCHAR(256) NOT NULL,
createdAt BIGINT UNSIGNED NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW(3)) * 1000),
updatedAt BIGINT UNSIGNED NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW(3)) * 1000)
);

CREATE INDEX idx_status_createdAt ON background_jobs (status, createdAt);
CREATE INDEX idx_createdBy ON background_jobs (createdBy);
CREATE INDEX idx_status ON background_jobs (status);
CREATE INDEX idx_jobType ON background_jobs (jobType);
CREATE INDEX idx_updatedAt ON background_jobs (updatedAt);

-- rename executable -> basic for test suites
UPDATE test_suite
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.executable'),
'$.basic',
JSON_EXTRACT(json, '$.executable')
)
WHERE JSON_EXTRACT(json, '$.executable') IS NOT NULL;

-- rename executableEntityReference -> basicEntityReference for test suites
UPDATE test_suite
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.executableEntityReference'),
'$.basicEntityReference',
JSON_EXTRACT(json, '$.executableEntityReference')
)
WHERE JSON_EXTRACT(json, '$.executableEntityReference') IS NOT NULL;

-- clean up the testSuites
UPDATE test_case SET json = json_remove(json, '$.testSuites');

-- clean up the testSuites in the version history too
UPDATE entity_extension SET json = json_remove(json, '$.testSuites') WHERE jsonSchema = 'testCase';

Empty file.
46 changes: 46 additions & 0 deletions bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
-- add timestamp index for test case result reindex performance
CREATE INDEX idx_timestamp_desc ON data_quality_data_time_series (timestamp DESC);

CREATE TABLE background_jobs (
id BIGSERIAL PRIMARY KEY,
jobType VARCHAR(256) NOT NULL,
methodName VARCHAR(256) NOT NULL,
jobArgs JSONB NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'PENDING',
createdBy VARCHAR(256) NOT NULL,
createdAt BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT,
updatedAt BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT
);

CREATE INDEX idx_status_createdAt ON background_jobs (status, createdAt);
CREATE INDEX idx_createdBy ON background_jobs (createdBy);
CREATE INDEX idx_status ON background_jobs (status);
CREATE INDEX idx_jobType ON background_jobs (jobType);
CREATE INDEX idx_updatedAt ON background_jobs (updatedAt);

-- rename executable -> basic for test suites
UPDATE test_suite
SET json = jsonb_set(
json::jsonb #- '{executable}',
'{basic}',
(json #> '{executable}')::jsonb,
true
)
WHERE json #>> '{executable}' IS NOT NULL;

-- rename executableEntityReference -> basicEntityReference for test suites
UPDATE test_suite
SET json = jsonb_set(
json::jsonb #- '{executableEntityReference}',
'{basicEntityReference}',
(json #> '{executableEntityReference}')::jsonb,
true
)
WHERE json #>> '{executableEntityReference}' IS NOT NULL;

-- clean up the testSuites
UPDATE test_case SET json = json::jsonb #- '{testSuites}';

-- clean up the testSuites in the version history too
UPDATE entity_extension SET json = json::jsonb #- '{testSuites}' WHERE jsonSchema = 'testCase';

Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package org.openmetadata.annotations;

import com.fasterxml.jackson.databind.JsonNode;
import com.sun.codemodel.JAnnotationUse;
import com.sun.codemodel.JClass;
import com.sun.codemodel.JDefinedClass;
import com.sun.codemodel.JFieldVar;
import com.sun.codemodel.JMethod;
import java.lang.reflect.Field;
import java.util.TreeMap;
import org.jsonschema2pojo.AbstractAnnotator;

/** Add {@link Deprecated} annotation to generated Java classes */
public class DeprecatedAnnotator extends AbstractAnnotator {

/** Add {@link Deprecated} annotation to property fields */
@Override
public void propertyField(
JFieldVar field, JDefinedClass clazz, String propertyName, JsonNode propertyNode) {
super.propertyField(field, clazz, propertyName, propertyNode);
if (propertyNode.get("deprecated") != null && propertyNode.get("deprecated").asBoolean()) {
field.annotate(Deprecated.class);
}
}

/** Add {@link Deprecated} annotation to getter methods */
@Override
public void propertyGetter(JMethod getter, JDefinedClass clazz, String propertyName) {
super.propertyGetter(getter, clazz, propertyName);
addDeprecatedAnnotationIfApplies(getter, propertyName);
}

/** Add {@link Deprecated} annotation to setter methods */
@Override
public void propertySetter(JMethod setter, JDefinedClass clazz, String propertyName) {
super.propertySetter(setter, clazz, propertyName);
addDeprecatedAnnotationIfApplies(setter, propertyName);
}

/**
* Use reflection methods to access the {@link JDefinedClass} of the {@link JMethod} object. If
* the {@link JMethod} is pointing to a field annotated with {@link Deprecated} then annotates
* the {@link JMethod} object with {@link Deprecated}
*/
private void addDeprecatedAnnotationIfApplies(JMethod jMethod, String propertyName) {
try {
Field outerClassField = JMethod.class.getDeclaredField("outer");
outerClassField.setAccessible(true);
JDefinedClass outerClass = (JDefinedClass) outerClassField.get(jMethod);

TreeMap<String, JFieldVar> insensitiveFieldsMap =
new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
insensitiveFieldsMap.putAll(outerClass.fields());

if (insensitiveFieldsMap.containsKey(propertyName)
&& insensitiveFieldsMap.get(propertyName).annotations().stream()
.anyMatch(
annotation ->
Deprecated.class.getName().equals(getAnnotationClassName(annotation)))) {
jMethod.annotate(Deprecated.class);
}
} catch (NoSuchFieldException | IllegalAccessException e) {
throw new RuntimeException(e);
}
}

private String getAnnotationClassName(JAnnotationUse annotation) {
try {
Field clazzField = JAnnotationUse.class.getDeclaredField("clazz");
clazzField.setAccessible(true);
return ((JClass) clazzField.get(annotation)).fullName();
} catch (NoSuchFieldException | IllegalAccessException e) {
throw new RuntimeException(e);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ public class OpenMetadataAnnotator extends CompositeAnnotator {

public OpenMetadataAnnotator() {
// we can add multiple annotators
super(new ExposedAnnotator(), new MaskedAnnotator(), new PasswordAnnotator());
super(
new ExposedAnnotator(),
new MaskedAnnotator(),
new PasswordAnnotator(),
new DeprecatedAnnotator());
}
}
10 changes: 8 additions & 2 deletions ingestion/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
"giturlparse": "giturlparse",
"validators": "validators~=0.22.0",
"teradata": "teradatasqlalchemy>=20.0.0.0",
"cockroach": "sqlalchemy-cockroachdb~=2.0",
"cassandra": "cassandra-driver>=3.28.0",
}

Expand All @@ -77,7 +78,7 @@
},
"kafka": {
VERSIONS["avro"],
"confluent_kafka==2.1.1",
"confluent_kafka>=2.1.1,<=2.6.1",
"fastavro>=1.2.0",
# Due to https://github.com/grpc/grpc/issues/30843#issuecomment-1303816925
# use >= v1.47.2 https://github.com/grpc/grpc/blob/v1.47.2/tools/distrib/python/grpcio_tools/grpc_version.py#L17
Expand Down Expand Up @@ -138,7 +139,7 @@
"requests>=2.23",
"requests-aws4auth~=1.1", # Only depends on requests as external package. Leaving as base.
"sqlalchemy>=1.4.0,<2",
"collate-sqllineage~=1.5.0",
"collate-sqllineage~=1.6.0",
"tabulate==0.9.0",
"typing-inspect",
"packaging", # For version parsing
Expand Down Expand Up @@ -233,6 +234,10 @@
"glue": {VERSIONS["boto3"]},
"great-expectations": {VERSIONS["great-expectations"]},
"greenplum": {*COMMONS["postgres"]},
"cockroach": {
VERSIONS["cockroach"],
"psycopg2-binary",
},
"hive": {
*COMMONS["hive"],
"thrift>=0.13,<1",
Expand Down Expand Up @@ -380,6 +385,7 @@
VERSIONS["avro"], # Sample Data
VERSIONS["grpc-tools"],
VERSIONS["neo4j"],
VERSIONS["cockroach"],
"testcontainers==3.7.1;python_version<'3.9'",
"testcontainers~=4.8.0;python_version>='3.9'",
"minio==7.2.5",
Expand Down
36 changes: 30 additions & 6 deletions ingestion/src/metadata/automations/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
from metadata.generated.schema.entity.automations.workflow import (
Workflow as AutomationWorkflow,
)
from metadata.ingestion.connections.test_connections import (
raise_test_connection_exception,
)
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
from metadata.utils.ssl_manager import SSLManager, check_ssl_and_init
Expand Down Expand Up @@ -60,19 +63,40 @@ def _(
"""
Run the test connection
"""

ssl_manager = None
ssl_manager: SSLManager = check_ssl_and_init(request.connection.config)
if ssl_manager:
request.connection.config = ssl_manager.setup_ssl(request.connection.config)

connection = get_connection(request.connection.config)

# Find the test_connection function in each <source>/connection.py file
test_connection_fn = get_test_connection_fn(request.connection.config)
test_connection_fn(
metadata, connection, request.connection.config, automation_workflow
)

try:
connection = get_connection(request.connection.config)

host_port_str = str(request.connection.config.hostPort or "")
if "localhost" in host_port_str:
result = test_connection_fn(metadata, connection, request.connection.config)
raise_test_connection_exception(result)

test_connection_fn(
metadata, connection, request.connection.config, automation_workflow
)
except Exception as error:
host_port_str = str(getattr(request.connection.config, "hostPort", None) or "")
if "localhost" not in host_port_str:
raise error

host_port_type = type(request.connection.config.hostPort)
docker_host_port_str = host_port_str.replace(
"localhost", "host.docker.internal"
)
request.connection.config.hostPort = host_port_type(docker_host_port_str)

connection = get_connection(request.connection.config)
test_connection_fn(
metadata, connection, request.connection.config, automation_workflow
)

if ssl_manager:
ssl_manager.cleanup_temp_files()
Loading

0 comments on commit 83a6e39

Please sign in to comment.