diff --git a/docker-compose-web-app.yml b/docker-compose-web-app.yml index 7d1b768ef37..f7d7031d6f7 100644 --- a/docker-compose-web-app.yml +++ b/docker-compose-web-app.yml @@ -20,7 +20,7 @@ version: "3.8" services: # Frontend service community-app: - image: openmf/web-app:latest + image: openmf/web-app:master container_name: mifos-web-app restart: always ports: diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/DataSourcePerTenantServiceFactory.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/DataSourcePerTenantServiceFactory.java index ea980e5164d..f628071683a 100644 --- a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/DataSourcePerTenantServiceFactory.java +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/DataSourcePerTenantServiceFactory.java @@ -23,46 +23,40 @@ import com.zaxxer.hikari.HikariConfig; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.micrometer.core.instrument.MeterRegistry; import javax.sql.DataSource; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.fineract.infrastructure.core.config.FineractProperties; +import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant; import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection; +import org.apache.fineract.infrastructure.core.service.database.metrics.TenantConnectionPoolMetricsTrackerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; /** - * * Factory class to get data source service based on the details stored in {@link FineractPlatformTenantConnection} * variable - * */ @Component @Slf4j +@RequiredArgsConstructor public class DataSourcePerTenantServiceFactory { private final HikariConfig hikariConfig; private final FineractProperties fineractProperties; private final ApplicationContext context; + @Qualifier("hikariTenantDataSource") private final DataSource tenantDataSource; private final HikariDataSourceFactory hikariDataSourceFactory; private final DatabasePasswordEncryptor databasePasswordEncryptor; - - public DataSourcePerTenantServiceFactory(@Qualifier("hikariTenantDataSource") DataSource tenantDataSource, HikariConfig hikariConfig, - FineractProperties fineractProperties, ApplicationContext context, HikariDataSourceFactory hikariDataSourceFactory, - DatabasePasswordEncryptor databasePasswordEncryptor) { - this.hikariConfig = hikariConfig; - this.fineractProperties = fineractProperties; - this.context = context; - this.tenantDataSource = tenantDataSource; - this.hikariDataSourceFactory = hikariDataSourceFactory; - this.databasePasswordEncryptor = databasePasswordEncryptor; - } + private final MeterRegistry meterRegistry; @SuppressFBWarnings(value = "SLF4J_SIGN_ONLY_FORMAT") - public DataSource createNewDataSourceFor(final FineractPlatformTenantConnection tenantConnection) { + public DataSource createNewDataSourceFor(FineractPlatformTenant tenant, FineractPlatformTenantConnection tenantConnection) { if (!databasePasswordEncryptor.isMasterPasswordHashValid(tenantConnection.getMasterPasswordHash())) { throw new IllegalArgumentException( "Invalid master password on tenant connection %d.".formatted(tenantConnection.getConnectionId())); @@ -103,6 +97,7 @@ public DataSource createNewDataSourceFor(final FineractPlatformTenantConnection // https://github.com/brettwooldridge/HikariCP/wiki/MBean-(JMX)-Monitoring-and-Management config.setRegisterMbeans(true); + config.setMetricsTrackerFactory(new TenantConnectionPoolMetricsTrackerFactory(tenant.getTenantIdentifier(), meterRegistry)); // https://github.com/brettwooldridge/HikariCP/wiki/MySQL-Configuration // These are the properties for each Tenant DB; the same configuration diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/TomcatJdbcDataSourcePerTenantService.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/TomcatJdbcDataSourcePerTenantService.java index 9e11be36c8f..e3b792e1b11 100644 --- a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/TomcatJdbcDataSourcePerTenantService.java +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/TomcatJdbcDataSourcePerTenantService.java @@ -24,12 +24,12 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.sql.DataSource; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant; import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.apache.fineract.infrastructure.core.service.tenant.TenantDetailsService; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextRefreshedEvent; @@ -43,22 +43,16 @@ */ @Slf4j @Service +@RequiredArgsConstructor public class TomcatJdbcDataSourcePerTenantService implements RoutingDataSourceService, ApplicationListener { private static final Map TENANT_TO_DATA_SOURCE_MAP = new ConcurrentHashMap<>(); + @Qualifier("hikariTenantDataSource") private final DataSource tenantDataSource; private final TenantDetailsService tenantDetailsService; private final DataSourcePerTenantServiceFactory dataSourcePerTenantServiceFactory; - @Autowired - public TomcatJdbcDataSourcePerTenantService(final @Qualifier("hikariTenantDataSource") DataSource tenantDataSource, - final DataSourcePerTenantServiceFactory dataSourcePerTenantServiceFactory, final TenantDetailsService tenantDetailsService) { - this.tenantDataSource = tenantDataSource; - this.dataSourcePerTenantServiceFactory = dataSourcePerTenantServiceFactory; - this.tenantDetailsService = tenantDetailsService; - } - @Override public DataSource retrieveDataSource() { // default to tenant database datasource @@ -71,7 +65,7 @@ public DataSource retrieveDataSource() { // if tenantConnection information available switch to the // appropriate datasource for that tenant. actualDataSource = TENANT_TO_DATA_SOURCE_MAP.computeIfAbsent(tenantConnectionKey, - (key) -> dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenantConnection)); + (key) -> dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenant, tenantConnection)); } @@ -91,7 +85,7 @@ private void initializeDataSourceConnection(FineractPlatformTenant tenant) { final FineractPlatformTenantConnection tenantConnection = tenant.getConnection(); Long tenantConnectionKey = tenantConnection.getConnectionId(); TENANT_TO_DATA_SOURCE_MAP.computeIfAbsent(tenantConnectionKey, (key) -> { - DataSource tenantSpecificDataSource = dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenantConnection); + DataSource tenantSpecificDataSource = dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenant, tenantConnection); try (Connection connection = tenantSpecificDataSource.getConnection()) { String url = connection.getMetaData().getURL(); log.debug("Established database connection with URL {}", url); diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTracker.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTracker.java new file mode 100644 index 00000000000..9b1f1682299 --- /dev/null +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTracker.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fineract.infrastructure.core.service.database.metrics; + +import com.zaxxer.hikari.metrics.IMetricsTracker; +import com.zaxxer.hikari.metrics.PoolStats; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import java.util.concurrent.TimeUnit; + +public class TenantConnectionPoolMetricsTracker implements IMetricsTracker { + + public static final String HIKARI_METRIC_NAME_PREFIX = ".hikaricp"; + + private static final String METRIC_CATEGORY = "pool"; + private static final String METRIC_NAME_WAIT = HIKARI_METRIC_NAME_PREFIX + ".connections.acquire"; + private static final String METRIC_NAME_USAGE = HIKARI_METRIC_NAME_PREFIX + ".connections.usage"; + private static final String METRIC_NAME_CONNECT = HIKARI_METRIC_NAME_PREFIX + ".connections.creation"; + + private static final String METRIC_NAME_TIMEOUT_RATE = HIKARI_METRIC_NAME_PREFIX + ".connections.timeout"; + private static final String METRIC_NAME_TOTAL_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections"; + private static final String METRIC_NAME_IDLE_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.idle"; + private static final String METRIC_NAME_ACTIVE_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.active"; + private static final String METRIC_NAME_PENDING_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.pending"; + private static final String METRIC_NAME_MAX_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.max"; + private static final String METRIC_NAME_MIN_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.min"; + + private final Timer connectionObtainTimer; + private final Counter connectionTimeoutCounter; + private final Timer connectionUsage; + private final Timer connectionCreation; + private final Gauge totalConnectionGauge; + private final Gauge idleConnectionGauge; + private final Gauge activeConnectionGauge; + private final Gauge pendingConnectionGauge; + private final Gauge maxConnectionGauge; + private final Gauge minConnectionGauge; + private final MeterRegistry meterRegistry; + private final PoolStats poolStats; + + public TenantConnectionPoolMetricsTracker(String tenantIdentifier, String poolName, PoolStats poolStats, MeterRegistry meterRegistry) { + // poolStats must be held with a 'strong reference' even though it is never referenced within this class + this.poolStats = poolStats; // DO NOT REMOVE + + this.meterRegistry = meterRegistry; + + String metricPrefix = "fineract.tenants." + tenantIdentifier; + + this.connectionObtainTimer = Timer.builder(metricPrefix + METRIC_NAME_WAIT).description("Connection acquire time") // + .tags(METRIC_CATEGORY, poolName) // + .register(meterRegistry); + + this.connectionCreation = Timer.builder(metricPrefix + METRIC_NAME_CONNECT).description("Connection creation time") // + .tags(METRIC_CATEGORY, poolName) // + .register(meterRegistry); + + this.connectionUsage = Timer.builder(metricPrefix + METRIC_NAME_USAGE).description("Connection usage time") // + .tags(METRIC_CATEGORY, poolName) // + .register(meterRegistry); + + this.connectionTimeoutCounter = Counter.builder(metricPrefix + METRIC_NAME_TIMEOUT_RATE) + .description("Connection timeout total count") // + .tags(METRIC_CATEGORY, poolName) // + .register(meterRegistry); + + this.totalConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_TOTAL_CONNECTIONS, poolStats, PoolStats::getTotalConnections) + .description("Total connections") // + .tags(METRIC_CATEGORY, poolName) // + .register(meterRegistry); + + this.idleConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_IDLE_CONNECTIONS, poolStats, PoolStats::getIdleConnections) + .description("Idle connections") // + .tags(METRIC_CATEGORY, poolName) // + .register(meterRegistry); + + this.activeConnectionGauge = Gauge + .builder(metricPrefix + METRIC_NAME_ACTIVE_CONNECTIONS, poolStats, PoolStats::getActiveConnections) + .description("Active connections") // + .tags(METRIC_CATEGORY, poolName) // + .register(meterRegistry); + + this.pendingConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_PENDING_CONNECTIONS, poolStats, PoolStats::getPendingThreads) + .description("Pending threads") // + .tags(METRIC_CATEGORY, poolName) // + .register(meterRegistry); + + this.maxConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_MAX_CONNECTIONS, poolStats, PoolStats::getMaxConnections) + .description("Max connections") // + .tags(METRIC_CATEGORY, poolName) // + .register(meterRegistry); + + this.minConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_MIN_CONNECTIONS, poolStats, PoolStats::getMinConnections) + .description("Min connections") // + .tags(METRIC_CATEGORY, poolName) // + .register(meterRegistry); + + } + + @Override + public void recordConnectionAcquiredNanos(final long elapsedAcquiredNanos) { + connectionObtainTimer.record(elapsedAcquiredNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void recordConnectionUsageMillis(final long elapsedBorrowedMillis) { + connectionUsage.record(elapsedBorrowedMillis, TimeUnit.MILLISECONDS); + } + + @Override + public void recordConnectionTimeout() { + connectionTimeoutCounter.increment(); + } + + @Override + public void recordConnectionCreatedMillis(long connectionCreatedMillis) { + connectionCreation.record(connectionCreatedMillis, TimeUnit.MILLISECONDS); + } + + @Override + public void close() { + meterRegistry.remove(connectionObtainTimer); + meterRegistry.remove(connectionTimeoutCounter); + meterRegistry.remove(connectionUsage); + meterRegistry.remove(connectionCreation); + meterRegistry.remove(totalConnectionGauge); + meterRegistry.remove(idleConnectionGauge); + meterRegistry.remove(activeConnectionGauge); + meterRegistry.remove(pendingConnectionGauge); + meterRegistry.remove(maxConnectionGauge); + meterRegistry.remove(minConnectionGauge); + } +} diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTrackerFactory.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTrackerFactory.java new file mode 100644 index 00000000000..6a7631db49f --- /dev/null +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTrackerFactory.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.core.service.database.metrics; + +import com.zaxxer.hikari.metrics.IMetricsTracker; +import com.zaxxer.hikari.metrics.MetricsTrackerFactory; +import com.zaxxer.hikari.metrics.PoolStats; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class TenantConnectionPoolMetricsTrackerFactory implements MetricsTrackerFactory { + + private final String tenantIdentifier; + private final MeterRegistry registry; + + @Override + public IMetricsTracker create(String poolName, PoolStats poolStats) { + return new TenantConnectionPoolMetricsTracker(tenantIdentifier, poolName, poolStats, registry); + } +} diff --git a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/core/DataSourcePerTenantServiceFactoryTest.java b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/core/DataSourcePerTenantServiceFactoryTest.java index 5ead9f5f0cc..546bc541a53 100644 --- a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/core/DataSourcePerTenantServiceFactoryTest.java +++ b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/core/DataSourcePerTenantServiceFactoryTest.java @@ -87,6 +87,7 @@ public class DataSourcePerTenantServiceFactoryTest { public static final String MASTER_MASTER_PASSWORD = "fineract"; public static final String MASTER_ENCRYPTION = "AES/CBC/PKCS5Padding"; + public static final FineractPlatformTenant TENANT = new FineractPlatformTenant(1L, "", "", "", null); @Mock private FineractProperties fineractProperties; @@ -183,7 +184,7 @@ void testCreateNewDataSourceFor_ShouldUseNormalConfiguration_WhenInAllMode() { given(fineractProperties.getMode()).willReturn(modeProperties); // when - DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection()); + DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection()); // then assertNotNull(dataSource); @@ -215,7 +216,7 @@ void testCreateNewDataSourceFor_ShouldOverridesMinPoolConfiguration_WhenConfigur config.setMinPoolSize(minPoolSize); // when - DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection()); + DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection()); // then assertNotNull(dataSource); @@ -247,7 +248,7 @@ void testCreateNewDataSourceFor_ShouldOverridesMaxPoolConfiguration_WhenConfigur config.setMaxPoolSize(maxPoolSize); // when - DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection()); + DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection()); // then assertNotNull(dataSource); @@ -281,7 +282,7 @@ void testCreateNewDataSourceFor_ShouldOverridesMinAndMaxPoolConfiguration_WhenBo config.setMaxPoolSize(maxPoolSize); // when - DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection()); + DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection()); // then assertNotNull(dataSource); @@ -307,7 +308,7 @@ void testCreateNewDataSourceFor_ShouldUseReadOnlyConfiguration_WhenInReadOnlyMod given(fineractProperties.getMode()).willReturn(modeProperties); // when - DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection()); + DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection()); // then assertNotNull(dataSource); @@ -333,7 +334,7 @@ void testCreateNewDataSourceFor_ShouldUseNormalConfiguration_WhenInBatchOnlyMode given(fineractProperties.getMode()).willReturn(modeProperties); // when - DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection()); + DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection()); // then assertNotNull(dataSource);