Skip to content

Commit

Permalink
FINERACT-2166: Expose tenant connection pool metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
galovics committed Jan 10, 2025
1 parent 81883a1 commit da5dad2
Show file tree
Hide file tree
Showing 6 changed files with 210 additions and 32 deletions.
2 changes: 1 addition & 1 deletion docker-compose-web-app.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ version: "3.8"
services:
# Frontend service
community-app:
image: openmf/web-app:latest
image: openmf/web-app:master
container_name: mifos-web-app
restart: always
ports:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,46 +23,40 @@

import com.zaxxer.hikari.HikariConfig;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.micrometer.core.instrument.MeterRegistry;
import javax.sql.DataSource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection;
import org.apache.fineract.infrastructure.core.service.database.metrics.TenantConnectionPoolMetricsTrackerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;

/**
*
* Factory class to get data source service based on the details stored in {@link FineractPlatformTenantConnection}
* variable
*
*/
@Component
@Slf4j
@RequiredArgsConstructor
public class DataSourcePerTenantServiceFactory {

private final HikariConfig hikariConfig;
private final FineractProperties fineractProperties;
private final ApplicationContext context;
@Qualifier("hikariTenantDataSource")
private final DataSource tenantDataSource;
private final HikariDataSourceFactory hikariDataSourceFactory;

private final DatabasePasswordEncryptor databasePasswordEncryptor;

public DataSourcePerTenantServiceFactory(@Qualifier("hikariTenantDataSource") DataSource tenantDataSource, HikariConfig hikariConfig,
FineractProperties fineractProperties, ApplicationContext context, HikariDataSourceFactory hikariDataSourceFactory,
DatabasePasswordEncryptor databasePasswordEncryptor) {
this.hikariConfig = hikariConfig;
this.fineractProperties = fineractProperties;
this.context = context;
this.tenantDataSource = tenantDataSource;
this.hikariDataSourceFactory = hikariDataSourceFactory;
this.databasePasswordEncryptor = databasePasswordEncryptor;
}
private final MeterRegistry meterRegistry;

@SuppressFBWarnings(value = "SLF4J_SIGN_ONLY_FORMAT")
public DataSource createNewDataSourceFor(final FineractPlatformTenantConnection tenantConnection) {
public DataSource createNewDataSourceFor(FineractPlatformTenant tenant, FineractPlatformTenantConnection tenantConnection) {
if (!databasePasswordEncryptor.isMasterPasswordHashValid(tenantConnection.getMasterPasswordHash())) {
throw new IllegalArgumentException(
"Invalid master password on tenant connection %d.".formatted(tenantConnection.getConnectionId()));
Expand Down Expand Up @@ -103,6 +97,7 @@ public DataSource createNewDataSourceFor(final FineractPlatformTenantConnection

// https://github.com/brettwooldridge/HikariCP/wiki/MBean-(JMX)-Monitoring-and-Management
config.setRegisterMbeans(true);
config.setMetricsTrackerFactory(new TenantConnectionPoolMetricsTrackerFactory(tenant.getTenantIdentifier(), meterRegistry));

// https://github.com/brettwooldridge/HikariCP/wiki/MySQL-Configuration
// These are the properties for each Tenant DB; the same configuration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.sql.DataSource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import org.apache.fineract.infrastructure.core.service.tenant.TenantDetailsService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationListener;
import org.springframework.context.event.ContextRefreshedEvent;
Expand All @@ -43,22 +43,16 @@
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class TomcatJdbcDataSourcePerTenantService implements RoutingDataSourceService, ApplicationListener<ContextRefreshedEvent> {

private static final Map<Long, DataSource> TENANT_TO_DATA_SOURCE_MAP = new ConcurrentHashMap<>();
@Qualifier("hikariTenantDataSource")
private final DataSource tenantDataSource;
private final TenantDetailsService tenantDetailsService;

private final DataSourcePerTenantServiceFactory dataSourcePerTenantServiceFactory;

@Autowired
public TomcatJdbcDataSourcePerTenantService(final @Qualifier("hikariTenantDataSource") DataSource tenantDataSource,
final DataSourcePerTenantServiceFactory dataSourcePerTenantServiceFactory, final TenantDetailsService tenantDetailsService) {
this.tenantDataSource = tenantDataSource;
this.dataSourcePerTenantServiceFactory = dataSourcePerTenantServiceFactory;
this.tenantDetailsService = tenantDetailsService;
}

@Override
public DataSource retrieveDataSource() {
// default to tenant database datasource
Expand All @@ -71,7 +65,7 @@ public DataSource retrieveDataSource() {
// if tenantConnection information available switch to the
// appropriate datasource for that tenant.
actualDataSource = TENANT_TO_DATA_SOURCE_MAP.computeIfAbsent(tenantConnectionKey,
(key) -> dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenantConnection));
(key) -> dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenant, tenantConnection));

}

Expand All @@ -91,7 +85,7 @@ private void initializeDataSourceConnection(FineractPlatformTenant tenant) {
final FineractPlatformTenantConnection tenantConnection = tenant.getConnection();
Long tenantConnectionKey = tenantConnection.getConnectionId();
TENANT_TO_DATA_SOURCE_MAP.computeIfAbsent(tenantConnectionKey, (key) -> {
DataSource tenantSpecificDataSource = dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenantConnection);
DataSource tenantSpecificDataSource = dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenant, tenantConnection);
try (Connection connection = tenantSpecificDataSource.getConnection()) {
String url = connection.getMetaData().getURL();
log.debug("Established database connection with URL {}", url);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.fineract.infrastructure.core.service.database.metrics;

import com.zaxxer.hikari.metrics.IMetricsTracker;
import com.zaxxer.hikari.metrics.PoolStats;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import java.util.concurrent.TimeUnit;

public class TenantConnectionPoolMetricsTracker implements IMetricsTracker {

public static final String HIKARI_METRIC_NAME_PREFIX = ".hikaricp";

private static final String METRIC_CATEGORY = "pool";
private static final String METRIC_NAME_WAIT = HIKARI_METRIC_NAME_PREFIX + ".connections.acquire";
private static final String METRIC_NAME_USAGE = HIKARI_METRIC_NAME_PREFIX + ".connections.usage";
private static final String METRIC_NAME_CONNECT = HIKARI_METRIC_NAME_PREFIX + ".connections.creation";

private static final String METRIC_NAME_TIMEOUT_RATE = HIKARI_METRIC_NAME_PREFIX + ".connections.timeout";
private static final String METRIC_NAME_TOTAL_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections";
private static final String METRIC_NAME_IDLE_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.idle";
private static final String METRIC_NAME_ACTIVE_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.active";
private static final String METRIC_NAME_PENDING_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.pending";
private static final String METRIC_NAME_MAX_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.max";
private static final String METRIC_NAME_MIN_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.min";

private final Timer connectionObtainTimer;
private final Counter connectionTimeoutCounter;
private final Timer connectionUsage;
private final Timer connectionCreation;
private final Gauge totalConnectionGauge;
private final Gauge idleConnectionGauge;
private final Gauge activeConnectionGauge;
private final Gauge pendingConnectionGauge;
private final Gauge maxConnectionGauge;
private final Gauge minConnectionGauge;
private final MeterRegistry meterRegistry;
private final PoolStats poolStats;

public TenantConnectionPoolMetricsTracker(String tenantIdentifier, String poolName, PoolStats poolStats, MeterRegistry meterRegistry) {
// poolStats must be held with a 'strong reference' even though it is never referenced within this class
this.poolStats = poolStats; // DO NOT REMOVE

this.meterRegistry = meterRegistry;

String metricPrefix = "fineract.tenants." + tenantIdentifier;

this.connectionObtainTimer = Timer.builder(metricPrefix + METRIC_NAME_WAIT).description("Connection acquire time") //
.tags(METRIC_CATEGORY, poolName) //
.register(meterRegistry);

this.connectionCreation = Timer.builder(metricPrefix + METRIC_NAME_CONNECT).description("Connection creation time") //
.tags(METRIC_CATEGORY, poolName) //
.register(meterRegistry);

this.connectionUsage = Timer.builder(metricPrefix + METRIC_NAME_USAGE).description("Connection usage time") //
.tags(METRIC_CATEGORY, poolName) //
.register(meterRegistry);

this.connectionTimeoutCounter = Counter.builder(metricPrefix + METRIC_NAME_TIMEOUT_RATE)
.description("Connection timeout total count") //
.tags(METRIC_CATEGORY, poolName) //
.register(meterRegistry);

this.totalConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_TOTAL_CONNECTIONS, poolStats, PoolStats::getTotalConnections)
.description("Total connections") //
.tags(METRIC_CATEGORY, poolName) //
.register(meterRegistry);

this.idleConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_IDLE_CONNECTIONS, poolStats, PoolStats::getIdleConnections)
.description("Idle connections") //
.tags(METRIC_CATEGORY, poolName) //
.register(meterRegistry);

this.activeConnectionGauge = Gauge
.builder(metricPrefix + METRIC_NAME_ACTIVE_CONNECTIONS, poolStats, PoolStats::getActiveConnections)
.description("Active connections") //
.tags(METRIC_CATEGORY, poolName) //
.register(meterRegistry);

this.pendingConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_PENDING_CONNECTIONS, poolStats, PoolStats::getPendingThreads)
.description("Pending threads") //
.tags(METRIC_CATEGORY, poolName) //
.register(meterRegistry);

this.maxConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_MAX_CONNECTIONS, poolStats, PoolStats::getMaxConnections)
.description("Max connections") //
.tags(METRIC_CATEGORY, poolName) //
.register(meterRegistry);

this.minConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_MIN_CONNECTIONS, poolStats, PoolStats::getMinConnections)
.description("Min connections") //
.tags(METRIC_CATEGORY, poolName) //
.register(meterRegistry);

}

@Override
public void recordConnectionAcquiredNanos(final long elapsedAcquiredNanos) {
connectionObtainTimer.record(elapsedAcquiredNanos, TimeUnit.NANOSECONDS);
}

@Override
public void recordConnectionUsageMillis(final long elapsedBorrowedMillis) {
connectionUsage.record(elapsedBorrowedMillis, TimeUnit.MILLISECONDS);
}

@Override
public void recordConnectionTimeout() {
connectionTimeoutCounter.increment();
}

@Override
public void recordConnectionCreatedMillis(long connectionCreatedMillis) {
connectionCreation.record(connectionCreatedMillis, TimeUnit.MILLISECONDS);
}

@Override
public void close() {
meterRegistry.remove(connectionObtainTimer);
meterRegistry.remove(connectionTimeoutCounter);
meterRegistry.remove(connectionUsage);
meterRegistry.remove(connectionCreation);
meterRegistry.remove(totalConnectionGauge);
meterRegistry.remove(idleConnectionGauge);
meterRegistry.remove(activeConnectionGauge);
meterRegistry.remove(pendingConnectionGauge);
meterRegistry.remove(maxConnectionGauge);
meterRegistry.remove(minConnectionGauge);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.database.metrics;

import com.zaxxer.hikari.metrics.IMetricsTracker;
import com.zaxxer.hikari.metrics.MetricsTrackerFactory;
import com.zaxxer.hikari.metrics.PoolStats;
import io.micrometer.core.instrument.MeterRegistry;
import lombok.RequiredArgsConstructor;

@RequiredArgsConstructor
public class TenantConnectionPoolMetricsTrackerFactory implements MetricsTrackerFactory {

private final String tenantIdentifier;
private final MeterRegistry registry;

@Override
public IMetricsTracker create(String poolName, PoolStats poolStats) {
return new TenantConnectionPoolMetricsTracker(tenantIdentifier, poolName, poolStats, registry);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ public class DataSourcePerTenantServiceFactoryTest {
public static final String MASTER_MASTER_PASSWORD = "fineract";

public static final String MASTER_ENCRYPTION = "AES/CBC/PKCS5Padding";
public static final FineractPlatformTenant TENANT = new FineractPlatformTenant(1L, "", "", "", null);

@Mock
private FineractProperties fineractProperties;
Expand Down Expand Up @@ -183,7 +184,7 @@ void testCreateNewDataSourceFor_ShouldUseNormalConfiguration_WhenInAllMode() {
given(fineractProperties.getMode()).willReturn(modeProperties);

// when
DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());

// then
assertNotNull(dataSource);
Expand Down Expand Up @@ -215,7 +216,7 @@ void testCreateNewDataSourceFor_ShouldOverridesMinPoolConfiguration_WhenConfigur
config.setMinPoolSize(minPoolSize);

// when
DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());

// then
assertNotNull(dataSource);
Expand Down Expand Up @@ -247,7 +248,7 @@ void testCreateNewDataSourceFor_ShouldOverridesMaxPoolConfiguration_WhenConfigur
config.setMaxPoolSize(maxPoolSize);

// when
DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());

// then
assertNotNull(dataSource);
Expand Down Expand Up @@ -281,7 +282,7 @@ void testCreateNewDataSourceFor_ShouldOverridesMinAndMaxPoolConfiguration_WhenBo
config.setMaxPoolSize(maxPoolSize);

// when
DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());

// then
assertNotNull(dataSource);
Expand All @@ -307,7 +308,7 @@ void testCreateNewDataSourceFor_ShouldUseReadOnlyConfiguration_WhenInReadOnlyMod
given(fineractProperties.getMode()).willReturn(modeProperties);

// when
DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());

// then
assertNotNull(dataSource);
Expand All @@ -333,7 +334,7 @@ void testCreateNewDataSourceFor_ShouldUseNormalConfiguration_WhenInBatchOnlyMode
given(fineractProperties.getMode()).willReturn(modeProperties);

// when
DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());

// then
assertNotNull(dataSource);
Expand Down

0 comments on commit da5dad2

Please sign in to comment.