Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix #18834: Introduce new setting OpenMetadataBaseUrlConfiguration to set the server endpoint #19366

Merged
merged 20 commits into from
Jan 27, 2025
Merged
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
33f1a8f
Refactor: Introduce openMetadataBaseUrlConfiguration to set up the se…
Siddhanttimeline Jan 15, 2025
a9e12af
migrations.
Siddhanttimeline Jan 15, 2025
9b58d13
Merge branch 'main' into fix-18834
Siddhanttimeline Jan 15, 2025
4394c22
Merge branch 'main' into fix-18834
Siddhanttimeline Jan 16, 2025
56eda60
Merge branch 'main' into fix-18834
Siddhanttimeline Jan 17, 2025
048599a
refactor(cli): add setOpenMetadataUrl command to store OpenMetadata URL
Siddhanttimeline Jan 17, 2025
9a66a9d
refactor(cli): add configureEmailSettings command to manage SMTP/Emai…
Siddhanttimeline Jan 17, 2025
126c6f3
load initialData when config is not present in the db
Siddhanttimeline Jan 17, 2025
c6fadb2
Merge branch 'main' into fix-18834
Siddhanttimeline Jan 17, 2025
5adcac7
Merge branch 'main' into fix-18834
Siddhanttimeline Jan 23, 2025
dd221f7
Merge branch 'main' into fix-18834
Siddhanttimeline Jan 23, 2025
cb17ae9
Merge branch 'main' into fix-18834
karanh37 Jan 23, 2025
f62abbe
add om url settings page
karanh37 Jan 24, 2025
78e0644
update locales
karanh37 Jan 24, 2025
25fbff0
Merge branch 'main' into fix-18834
Siddhanttimeline Jan 24, 2025
32e4901
add e2e
karanh37 Jan 24, 2025
6335ca4
load omBaseUrl with default host and port: localhost:8585 when not pr…
Siddhanttimeline Jan 24, 2025
4c24d42
Merge remote-tracking branch 'origin/fix-18834' into fix-18834
Siddhanttimeline Jan 24, 2025
ced8210
Merge branch 'main' into fix-18834
Siddhanttimeline Jan 24, 2025
cc59628
Merge branch 'main' into fix-18834
Siddhanttimeline Jan 26, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor(cli): add configureEmailSettings command to manage SMTP/Emai…
…l configuration
Siddhanttimeline committed Jan 17, 2025

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 9a66a9dacd6f7bff0ec6d633c873f4d4fbabe9cf
12 changes: 0 additions & 12 deletions conf/openmetadata.yaml
Original file line number Diff line number Diff line change
@@ -370,18 +370,6 @@ health:
failureAttempts: 2
successAttempts: 1

email:
emailingEntity: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
supportUrl: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
enableSmtpServer : ${AUTHORIZER_ENABLE_SMTP:-false}
senderMail: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
serverEndpoint: ${SMTP_SERVER_ENDPOINT:-""}
serverPort: ${SMTP_SERVER_PORT:-""}
username: ${SMTP_SERVER_USERNAME:-""}
password: ${SMTP_SERVER_PWD:-""}
transportationStrategy: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
templates: ${TEMPLATES:-"openmetadata"}

limits:
enable: ${LIMITS_ENABLED:-false}
className: ${LIMITS_CLASS_NAME:-"org.openmetadata.service.limits.DefaultLimits"}
Original file line number Diff line number Diff line change
@@ -31,7 +31,6 @@
import org.openmetadata.schema.api.security.AuthorizerConfiguration;
import org.openmetadata.schema.api.security.jwt.JWTTokenConfiguration;
import org.openmetadata.schema.configuration.LimitsConfiguration;
import org.openmetadata.schema.email.SmtpSettings;
import org.openmetadata.schema.security.secrets.SecretsManagerConfiguration;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.service.config.OMWebConfiguration;
@@ -102,9 +101,6 @@ public PipelineServiceClientConfiguration getPipelineServiceClientConfiguration(
@JsonProperty("clusterName")
private String clusterName;

@JsonProperty("email")
private SmtpSettings smtpSettings;

@Valid
@NotNull
@JsonProperty("web")
Original file line number Diff line number Diff line change
@@ -79,7 +79,7 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app
Settings storedSettings = systemRepository.getConfigWithKey(EMAIL_CONFIGURATION.toString());
if (storedSettings == null) {
// Only in case a config doesn't exist in DB we insert it
SmtpSettings emailConfig = applicationConfig.getSmtpSettings();
SmtpSettings emailConfig = new SmtpSettings().withPassword(StringUtils.EMPTY);
Settings setting =
new Settings().withConfigType(EMAIL_CONFIGURATION).withConfigValue(emailConfig);
systemRepository.createNewSetting(setting);
Original file line number Diff line number Diff line change
@@ -255,30 +255,6 @@ public Response patch(
return systemRepository.patchSetting(settingName, patch);
}

@PUT
@Path("/restore/default/email")
@Operation(
operationId = "restoreEmailSettingToDefault",
summary = "Restore Email to Default setting",
description = "Restore Email to Default settings",
responses = {
@ApiResponse(
responseCode = "200",
description = "Settings",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Settings.class)))
})
public Response restoreDefaultEmailSetting(
@Context UriInfo uriInfo, @Context SecurityContext securityContext) {
authorizer.authorizeAdmin(securityContext);
return systemRepository.createOrUpdate(
new Settings()
.withConfigType(SettingsType.EMAIL_CONFIGURATION)
.withConfigValue(applicationConfig.getSmtpSettings()));
}

@GET
@Path("/entities/count")
@Operation(
Original file line number Diff line number Diff line change
@@ -42,6 +42,8 @@
import org.jdbi.v3.core.Jdbi;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.ServiceEntityInterface;
import org.openmetadata.schema.api.configuration.OpenMetadataBaseUrlConfiguration;
import org.openmetadata.schema.email.SmtpSettings;
import org.openmetadata.schema.entity.app.App;
import org.openmetadata.schema.entity.app.AppConfiguration;
import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition;
@@ -176,45 +178,144 @@ public Integer repair() {
}

@Command(
name = "syncEmailFromEnv",
description = "Sync the email configuration from environment variables")
public Integer syncEmailFromEnv() {
name = "setOpenMetadataUrl",
description = "Set or update the OpenMetadata URL in the system repository")
public Integer setOpenMetadataUrl(
@Option(
names = {"-u", "--url"},
description = "OpenMetadata URL to store in the system repository",
required = true)
String openMetadataUrl) {
try {
parseConfig();
Settings updatedSettings =
new Settings()
.withConfigType(SettingsType.EMAIL_CONFIGURATION)
.withConfigValue(config.getSmtpSettings());
.withConfigType(SettingsType.OPEN_METADATA_BASE_URL_CONFIGURATION)
.withConfigValue(
new OpenMetadataBaseUrlConfiguration().withOpenMetadataUrl(openMetadataUrl));

Entity.getSystemRepository().createOrUpdate(updatedSettings);
LOG.info("Synced Email Configuration from Environment.");
LOG.info("Updated OpenMetadata URL to: {}", openMetadataUrl);
return 0;
} catch (Exception e) {
LOG.error("Email Sync failed due to ", e);
LOG.error("Failed to set OpenMetadata URL due to: ", e);
return 1;
}
}

@Command(
name = "setOpenMetadataUrl",
description = "Set or update the OpenMetadata URL in the system repository")
public Integer setOpenMetadataUrl(
name = "configureEmailSettings",
description =
"Set or update the SMTP/Email configuration in the OpenMetadata system repository")
public Integer configureEmailSettings(
@Option(
names = {"-u", "--url"},
description = "OpenMetadata URL to store in the system repository",
names = {"--emailingEntity"},
description = "Identifier or entity name used for sending emails (e.g. OpenMetadata)",
required = true)
String openMetadataUrl) {
String emailingEntity,
@Option(
names = {"--supportUrl"},
description =
"Support URL for help or documentation (e.g. https://slack.open-metadata.org)",
required = true)
String supportUrl,
@Option(
names = {"--enableSmtpServer"},
description = "Flag indicating whether SMTP server is enabled (true/false)",
required = true,
arity = "1")
boolean enableSmtpServer,
@Option(
names = {"--senderMail"},
description = "Sender email address used for outgoing messages",
required = true)
String senderMail,
@Option(
names = {"--serverEndpoint"},
description = "SMTP server endpoint (host)",
required = true)
String serverEndpoint,
@Option(
names = {"--serverPort"},
description = "SMTP server port",
required = true)
String serverPort,
@Option(
names = {"--username"},
description = "SMTP server username",
required = true)
String username,
@Option(
names = {"--password"},
description = "SMTP server password (may be masked)",
interactive = true,
arity = "0..1",
required = true)
char[] password,
@Option(
names = {"--transportationStrategy"},
description = "SMTP connection strategy (one of: SMTP, SMTPS, SMTP_TLS)",
required = true)
String transportationStrategy,
@Option(
names = {"--templatePath"},
description = "Custom path to email templates (if needed)")
String templatePath,
@Option(
names = {"--templates"},
description = "Email templates (e.g. openmetadata, collate)",
required = true)
String templates) {
try {
parseConfig();
Settings updatedSettings =

SmtpSettings smtpSettings = new SmtpSettings();
smtpSettings.setEmailingEntity(emailingEntity);
smtpSettings.setSupportUrl(supportUrl);
smtpSettings.setEnableSmtpServer(enableSmtpServer);
smtpSettings.setSenderMail(senderMail);
smtpSettings.setServerEndpoint(serverEndpoint);

smtpSettings.setServerPort(Integer.parseInt(serverPort));

smtpSettings.setUsername(username);
smtpSettings.setPassword(password != null ? new String(password) : "");

try {
smtpSettings.setTransportationStrategy(
SmtpSettings.TransportationStrategy.valueOf(transportationStrategy.toUpperCase()));
} catch (IllegalArgumentException e) {
LOG.warn(
"Invalid transportation strategy '{}'. Falling back to SMTP_TLS.",
transportationStrategy);
smtpSettings.setTransportationStrategy(SmtpSettings.TransportationStrategy.SMTP_TLS);
}

smtpSettings.setTemplatePath(templatePath);

try {
smtpSettings.setTemplates(SmtpSettings.Templates.valueOf(templates.toUpperCase()));
} catch (IllegalArgumentException e) {
LOG.warn("Invalid template value '{}'. Falling back to OPENMETADATA.", templates);
smtpSettings.setTemplates(SmtpSettings.Templates.OPENMETADATA);
}

Settings emailSettings =
new Settings()
.withConfigType(SettingsType.OPEN_METADATA_BASE_URL_CONFIGURATION)
.withConfigValue(openMetadataUrl);
.withConfigType(SettingsType.EMAIL_CONFIGURATION)
.withConfigValue(smtpSettings);

Entity.getSystemRepository().createOrUpdate(updatedSettings);
LOG.info("Updated OpenMetadata URL to: {}", openMetadataUrl);
Entity.getSystemRepository().createOrUpdate(emailSettings);

LOG.info(
"Email settings updated. (Email Entity: {}, SMTP Enabled: {}, SMTP Host: {})",
emailingEntity,
enableSmtpServer,
serverEndpoint);
return 0;

} catch (Exception e) {
LOG.error("Failed to set OpenMetadata URL due to: ", e);
LOG.error("Failed to configure email settings due to: ", e);
return 1;
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package org.openmetadata.service.resources.system;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.TEST_AUTH_HEADERS;

@@ -48,7 +47,6 @@
import org.openmetadata.schema.auth.JWTTokenExpiry;
import org.openmetadata.schema.configuration.AssetCertificationSettings;
import org.openmetadata.schema.configuration.WorkflowSettings;
import org.openmetadata.schema.email.SmtpSettings;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.profiler.MetricType;
@@ -58,10 +56,8 @@
import org.openmetadata.schema.type.ColumnDataType;
import org.openmetadata.schema.util.EntitiesCount;
import org.openmetadata.schema.util.ServicesCount;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.OpenMetadataApplicationTest;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.dashboards.DashboardResourceTest;
import org.openmetadata.service.resources.databases.TableResourceTest;
@@ -81,7 +77,6 @@
import org.openmetadata.service.resources.teams.TeamResourceTest;
import org.openmetadata.service.resources.teams.UserResourceTest;
import org.openmetadata.service.resources.topics.TopicResourceTest;
import org.openmetadata.service.secrets.masker.PasswordEntityMasker;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.TestUtils;

@@ -188,14 +183,6 @@ void entitiesCount(TestInfo test) throws HttpResponseException {
@Test
@Order(2)
void testSystemConfigs() throws HttpResponseException {
// Test Email Config
Settings emailSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION);
SmtpSettings smtp = JsonUtils.convertValue(emailSettings.getConfigValue(), SmtpSettings.class);
// Password for Email is always sent in hidden
SmtpSettings expected = config.getSmtpSettings();
expected.setPassword(PasswordEntityMasker.PASSWORD_MASK);
assertEquals(expected, smtp);

// Test Custom Ui Theme Preference Config
Settings uiThemeConfigWrapped = getSystemConfig(SettingsType.CUSTOM_UI_THEME_PREFERENCE);
UiThemePreference uiThemePreference =
@@ -211,40 +198,8 @@ void testSystemConfigs() throws HttpResponseException {
assertEquals("", uiThemePreference.getCustomLogoConfig().getCustomMonogramUrlPath());
}

@Test
@Order(1)
void testDefaultEmailSystemConfig() {
// Test Email Config
Settings stored =
Entity.getCollectionDAO()
.systemDAO()
.getConfigWithKey(SettingsType.EMAIL_CONFIGURATION.value());
SmtpSettings storedAndEncrypted =
JsonUtils.convertValue(stored.getConfigValue(), SmtpSettings.class);
assertTrue(Fernet.isTokenized(storedAndEncrypted.getPassword()));
assertEquals(
config.getSmtpSettings().getPassword(),
Fernet.getInstance().decryptIfApplies(storedAndEncrypted.getPassword()));
}

@Test
void testSystemConfigsUpdate(TestInfo test) throws HttpResponseException {
// Test Email Config
SmtpSettings smtpSettings = config.getSmtpSettings();
// Update a few Email fields
smtpSettings.setUsername(test.getDisplayName());
smtpSettings.setEmailingEntity(test.getDisplayName());

updateSystemConfig(
new Settings()
.withConfigType(SettingsType.EMAIL_CONFIGURATION)
.withConfigValue(smtpSettings));
SmtpSettings updateEmailSettings =
JsonUtils.convertValue(
getSystemConfig(SettingsType.EMAIL_CONFIGURATION).getConfigValue(), SmtpSettings.class);
assertEquals(updateEmailSettings.getUsername(), test.getDisplayName());
assertEquals(updateEmailSettings.getEmailingEntity(), test.getDisplayName());

// Test Custom Logo Update and theme preference
UiThemePreference updateConfigReq =
new UiThemePreference()
@@ -352,45 +307,13 @@ void validate_test() throws HttpResponseException {
@Test
void testDefaultSettingsInitialization() throws HttpResponseException {
SettingsCache.initialize(config);
Settings emailSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION);
Settings uiThemeSettings = getSystemConfig(SettingsType.CUSTOM_UI_THEME_PREFERENCE);
SmtpSettings smtpSettings =
JsonUtils.convertValue(emailSettings.getConfigValue(), SmtpSettings.class);
assertEquals(config.getSmtpSettings().getUsername(), smtpSettings.getUsername());
assertEquals(config.getSmtpSettings().getEmailingEntity(), smtpSettings.getEmailingEntity());
UiThemePreference uiThemePreference =
JsonUtils.convertValue(uiThemeSettings.getConfigValue(), UiThemePreference.class);
assertEquals("", uiThemePreference.getCustomTheme().getPrimaryColor());
assertEquals("", uiThemePreference.getCustomLogoConfig().getCustomLogoUrlPath());
}

@Test
void testEmailConfigurationSettings() throws HttpResponseException {
Settings emailSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION);
SmtpSettings smtpSettings =
JsonUtils.convertValue(emailSettings.getConfigValue(), SmtpSettings.class);
SmtpSettings expectedSmtpSettings = config.getSmtpSettings();
expectedSmtpSettings.setPassword(
smtpSettings.getPassword()); // Password is encrypted, so we use the stored one
assertEquals(expectedSmtpSettings, smtpSettings);
smtpSettings.setUsername("updatedUsername");
smtpSettings.setEmailingEntity("updatedEntity");

Settings updatedEmailSettings =
new Settings()
.withConfigType(SettingsType.EMAIL_CONFIGURATION)
.withConfigValue(smtpSettings);

updateSystemConfig(updatedEmailSettings);

Settings updatedSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION);
SmtpSettings updatedSmtpSettings =
JsonUtils.convertValue(updatedSettings.getConfigValue(), SmtpSettings.class);

assertEquals("updatedUsername", updatedSmtpSettings.getUsername());
assertEquals("updatedEntity", updatedSmtpSettings.getEmailingEntity());
}

@Order(3)
@Test
void testUiThemePreferenceSettings() throws HttpResponseException {
Loading