Skip to content

Commit

Permalink
Fix comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@IvanBorislavovDimitrov
IvanBorislavovDimitrov committed Jan 21, 2025
1 parent 0aaf113 commit fff4ae4
Show file tree
Hide file tree
Showing 8 changed files with 39 additions and 65 deletions.
10 changes: 5 additions & 5 deletions ...ps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,15 +78,15 @@ public final class Messages {
public static final String ERROR_OCCURRED_WHILE_CHECKING_FOR_INCREASED_LOCKS = "Error occurred while checking for increased locks";
public static final String THREAD_WAS_INTERRUPTED_WHILE_WAITING_FOR_THE_RESULT_OF_A_FUTURE = "Thread was interrupted while waiting for the result of a future";
public static final String ERROR_OCCURRED_DURING_HEALTH_CHECKING_FOR_INSTANCE_0_MESSAGE_1 = "Error occurred during health checking for instance: \"{0}\". Message: \"{1}\"";
public static final String OBJECT_STORE_FILE_STORAGE_HEALTH_DATABASE_HEALTH = "Object store file storage health: \"{}\", Database health: \"{}\"";
public static final String ERROR_OCCURRED_DURING_OBJECT_STORE_HEALTH_CHECKING_FOR_INSTANCE = "Error occurred during object store health checking for instance: \"{}\"";
public static final String OBJECT_STORE_FILE_STORAGE_HEALTH_DATABASE_HEALTH = "Object store file storage health: \"{0}\", Database health: \"{1}\"";
public static final String ERROR_OCCURRED_DURING_OBJECT_STORE_HEALTH_CHECKING_FOR_INSTANCE = "Error occurred during object store health checking for instance: \"{0}\"";
public static final String ERROR_OCCURRED_WHILE_CHECKING_DATABASE_INSTANCE_0 = "Error occurred while checking database instance: \"{0}\"";

// Warning messages
public static final String ENVIRONMENT_VARIABLE_IS_NOT_SET_USING_DEFAULT = "Environment variable \"{0}\" is not set. Using default \"{1}\"...";
public static final String OPTIONAL_RESOURCE_IS_NOT_SERVICE = "Optional resource \"{0}\" it will be not created because it''s not a service";
public static final String SERVICE_IS_NOT_ACTIVE = "Service \"{0}\" is inactive and will not be processed";
public static final String DETECTED_INCREASED_NUMBER_OF_PROCESSES_WAITING_FOR_LOCKS_FOR_INSTANCE = "Detected increased number of processes waiting for locks: \"{}\" for instance: \"{}\"";
public static final String DETECTED_INCREASED_NUMBER_OF_PROCESSES_WAITING_FOR_LOCKS_FOR_INSTANCE = "Detected increased number of processes waiting for locks: \"{0}\" for instance: \"{1}\"";
public static final String DETECTED_INCREASED_NUMBER_OF_PROCESSES_WAITING_FOR_LOCKS_FOR_INSTANCE_0_GETTING_THE_LOCKS = "Detected increased number of processes waiting for locks for instance {0}. Getting the locks...";

public static final String INVALID_VCAP_APPLICATION = "Invalid VCAP_APPLICATION \"{0}\"";
Expand Down Expand Up @@ -191,10 +191,10 @@ public final class Messages {
public static final String PARSED_TOKEN_TYPE_0 = "Parsed token type: {0}";
public static final String PARSED_TOKEN_EXPIRES_IN_0 = "Parsed token expires in: {0}";
public static final String PARSER_CHAIN_0 = "Parser chain: {0}";
public static final String VALUES_IN_INSTANCE_IN_THE_WAITING_FOR_LOCKS_SAMPLES = "Values in instance: \"{}\" in the waiting for locks samples: {}";
public static final String VALUES_IN_INSTANCE_IN_THE_WAITING_FOR_LOCKS_SAMPLES = "Values in instance: \"{0}\" in the waiting for locks samples: {1}";
public static final String INCREASING_OR_EQUAL_INDEX_0_1_2 = "Increasing or equal index: {0} / {1} = {2}";
public static final String DECREASING_INDEX_0_1_2 = "Decreasing index: {0} / {1} = {2}";
public static final String OBJECT_STORE_FILE_STORAGE_IS_NOT_AVAILABLE_FOR_INSTANCE = "Object store file storage is not available for instance: \"{}\"";
public static final String OBJECT_STORE_FILE_STORAGE_IS_NOT_AVAILABLE_FOR_INSTANCE = "Object store file storage is not available for instance: \"{0}\"";
public static final String NOT_ENOUGH_SAMPLES_TO_DETECT_ANOMALY_0_1 = "Not enough samples to detect anomaly: {0} / {1}";

// Audit log
Expand Down
33 changes: 20 additions & 13 deletions ...loudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@
@Named
public class ApplicationHealthCalculator {

private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationHealthCalculator.class.getName());
private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationHealthCalculator.class);

private static final int UPDATE_HEALTH_CHECK_STATUS_PERIOD_IN_SECONDS = 10;
private static final int TIMEOUT_FOR_TASK_EXECUTION_IN_SECONDS = 50;

private final ObjectStoreFileStorage objectStoreFileStorage;
private final ApplicationConfiguration applicationConfiguration;
Expand All @@ -49,6 +52,8 @@ public class ApplicationHealthCalculator {
private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
private final ExecutorService executor = Executors.newFixedThreadPool(3);

private final ResilientOperationExecutor resilientOperationExecutor = getResilienceExecutor();

@Autowired
public ApplicationHealthCalculator(@Autowired(required = false) ObjectStoreFileStorage objectStoreFileStorage,
ApplicationConfiguration applicationConfiguration, DatabaseHealthService databaseHealthService,
Expand All @@ -59,14 +64,14 @@ public ApplicationHealthCalculator(@Autowired(required = false) ObjectStoreFileS
this.databaseHealthService = databaseHealthService;
this.databaseMonitoringService = databaseMonitoringService;
this.databaseWaitingLocksAnalyzer = databaseWaitingLocksAnalyzer;
scheduler.scheduleAtFixedRate(this::updateHealthStatus, 0, 10, TimeUnit.SECONDS);
scheduler.scheduleAtFixedRate(this::updateHealthStatus, 0, UPDATE_HEALTH_CHECK_STATUS_PERIOD_IN_SECONDS, TimeUnit.SECONDS);
}

private void updateHealthStatus() {
List<Callable<Boolean>> tasks = List.of(this::isObjectStoreFileStorageHealthy, this::isDatabaseHealthy,
databaseWaitingLocksAnalyzer::hasIncreasedDbLocks);
try {
List<Future<Boolean>> completedFutures = executor.invokeAll(tasks, 50, TimeUnit.SECONDS);
List<Future<Boolean>> completedFutures = executor.invokeAll(tasks, TIMEOUT_FOR_TASK_EXECUTION_IN_SECONDS, TimeUnit.SECONDS);
executeFuture(completedFutures.get(0), isHealthy -> objectStoreFileStorageHealthCache.refresh(() -> isHealthy), false,
Messages.ERROR_OCCURRED_DURING_OBJECT_STORE_HEALTH_CHECKING);
executeFuture(completedFutures.get(1), isHealthy -> dbHealthServiceCache.refresh(() -> isHealthy), false,
Expand Down Expand Up @@ -111,7 +116,8 @@ public ResponseEntity<ApplicationHealthResult> calculateApplicationHealth() {
boolean isDbHealthy = dbHealthServiceCache.getOrRefresh(() -> false);

if (!isObjectStoreFileStorageHealthy || !isDbHealthy) {
LOGGER.error(Messages.OBJECT_STORE_FILE_STORAGE_HEALTH_DATABASE_HEALTH, isObjectStoreFileStorageHealthy, isDbHealthy);
LOGGER.error(MessageFormat.format(Messages.OBJECT_STORE_FILE_STORAGE_HEALTH_DATABASE_HEALTH, isObjectStoreFileStorageHealthy,
isDbHealthy));
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
.body(ImmutableApplicationHealthResult.builder()
.status(ApplicationHealthResult.Status.DOWN)
Expand All @@ -122,9 +128,9 @@ public ResponseEntity<ApplicationHealthResult> calculateApplicationHealth() {
if (hasIncreasedDbLocks) {
LOGGER.warn(Messages.DETECTED_INCREASED_NUMBER_OF_PROCESSES_WAITING_FOR_LOCKS_FOR_INSTANCE_0_GETTING_THE_LOCKS,
applicationConfiguration.getApplicationInstanceIndex());
long countOfProcessesWaitingForLocks = getResilienceExecutor().execute((Supplier<Long>) () -> databaseMonitoringService.getProcessesWaitingForLocks(ApplicationInstanceNameUtil.buildApplicationInstanceTemplate(applicationConfiguration)));
LOGGER.warn(Messages.DETECTED_INCREASED_NUMBER_OF_PROCESSES_WAITING_FOR_LOCKS_FOR_INSTANCE, countOfProcessesWaitingForLocks,
applicationConfiguration.getApplicationInstanceIndex());
long countOfProcessesWaitingForLocks = resilientOperationExecutor.execute((Supplier<Long>) () -> databaseMonitoringService.getProcessesWaitingForLocks(ApplicationInstanceNameUtil.buildApplicationInstanceTemplate(applicationConfiguration)));
LOGGER.warn(MessageFormat.format(Messages.DETECTED_INCREASED_NUMBER_OF_PROCESSES_WAITING_FOR_LOCKS_FOR_INSTANCE,
countOfProcessesWaitingForLocks, applicationConfiguration.getApplicationInstanceIndex()));
return ResponseEntity.ok(ImmutableApplicationHealthResult.builder() // TODO: Make this return 503 instead of 200 when the
// detection is trustworthy
.status(ApplicationHealthResult.Status.DOWN)
Expand All @@ -140,23 +146,24 @@ public ResponseEntity<ApplicationHealthResult> calculateApplicationHealth() {

private boolean isObjectStoreFileStorageHealthy() {
if (objectStoreFileStorage == null) {
LOGGER.debug(Messages.OBJECT_STORE_FILE_STORAGE_IS_NOT_AVAILABLE_FOR_INSTANCE,
applicationConfiguration.getApplicationInstanceIndex());
LOGGER.debug(MessageFormat.format(Messages.OBJECT_STORE_FILE_STORAGE_IS_NOT_AVAILABLE_FOR_INSTANCE,
applicationConfiguration.getApplicationInstanceIndex()));
return true;
}
try {
getResilienceExecutor().execute(objectStoreFileStorage::testConnection);
resilientOperationExecutor.execute(objectStoreFileStorage::testConnection);
} catch (Exception e) {
LOGGER.error(Messages.ERROR_OCCURRED_DURING_OBJECT_STORE_HEALTH_CHECKING_FOR_INSTANCE,
applicationConfiguration.getApplicationInstanceIndex(), e);
LOGGER.error(MessageFormat.format(Messages.ERROR_OCCURRED_DURING_OBJECT_STORE_HEALTH_CHECKING_FOR_INSTANCE,
applicationConfiguration.getApplicationInstanceIndex()),
e);
return false;
}
return true;
}

private boolean isDatabaseHealthy() {
try {
getResilienceExecutor().execute(databaseHealthService::testDatabaseConnection);
resilientOperationExecutor.execute(databaseHealthService::testDatabaseConnection);
return true;
} catch (Exception e) {
LOGGER.error(MessageFormat.format(Messages.ERROR_OCCURRED_WHILE_CHECKING_DATABASE_INSTANCE_0,
Expand Down
15 changes: 7 additions & 8 deletions ...y/multiapps/controller/core/application/health/database/DatabaseWaitingLocksAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ public class DatabaseWaitingLocksAnalyzer {
private static final int POLLING_LOCKS_INTERVAL_IN_SECONDS = 10;
private static final Duration MAXIMUM_VALIDITY_OF_LOCKS_SAMPLE_IN_MINUTES = Duration.ofMinutes(6);
private static final Duration ANOMALY_DETECTION_THRESHOLD_IN_MINUTES = Duration.ofMinutes(5);
public static final int MAXIMUM_VALUE_OF_NORMAL_LOCKS_COUNT = 5;
public static final double MAXIMAL_ACCEPTABLE_INCREMENTAL_LOCKS_DEVIATION_INDEX = 0.5;
private static final int MAXIMUM_VALUE_OF_NORMAL_LOCKS_COUNT = 5;
private static final double MAXIMAL_ACCEPTABLE_INCREMENTAL_LOCKS_DEVIATION_INDEX = 0.5;

private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
private final List<CachedObject<Long>> waitingLocksSamples = new LinkedList<>();
Expand All @@ -55,8 +55,7 @@ private void deleteObsoleteSamples() {

private void takeLocksSample() {
waitingLocksSamples.add(new CachedObject<>(databaseMonitoringService.getProcessesWaitingForLocks(ApplicationInstanceNameUtil.buildApplicationInstanceTemplate(applicationConfiguration)),
MAXIMUM_VALIDITY_OF_LOCKS_SAMPLE_IN_MINUTES,
System.currentTimeMillis() + MAXIMUM_VALIDITY_OF_LOCKS_SAMPLE_IN_MINUTES.toMillis()));
MAXIMUM_VALIDITY_OF_LOCKS_SAMPLE_IN_MINUTES));
}

public synchronized boolean hasIncreasedDbLocks() {
Expand All @@ -69,10 +68,10 @@ public synchronized boolean hasIncreasedDbLocks() {
boolean hasIncreasedLocks = calculateIncreasingOrEqualIndex() >= MAXIMAL_ACCEPTABLE_INCREMENTAL_LOCKS_DEVIATION_INDEX
&& checkIfLastOneThirdOfSequenceHasIncreasedOrIsEqualComparedToFirstOneThird(minimumRequiredSamplesCount);
if (hasIncreasedLocks) {
LOGGER.info(Messages.VALUES_IN_INSTANCE_IN_THE_WAITING_FOR_LOCKS_SAMPLES,
applicationConfiguration.getApplicationInstanceIndex(), waitingLocksSamples.stream()
.map(CachedObject::get)
.toList());
LOGGER.info(MessageFormat.format(Messages.VALUES_IN_INSTANCE_IN_THE_WAITING_FOR_LOCKS_SAMPLES,
applicationConfiguration.getApplicationInstanceIndex(), waitingLocksSamples.stream()
.map(CachedObject::get)
.toList()));
}
return hasIncreasedLocks;
}
Expand Down
9 changes: 0 additions & 9 deletions ...loudfoundry/multiapps/controller/core/application/health/database/WaitingLocksReport.java
View file

This file was deleted.

3 changes: 1 addition & 2 deletions ...roller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/model/CachedMap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,7 @@ public V computeIfAbsent(K key, Supplier<V> creator) {
}

public void put(K key, V value) {
long expirationTimestamp = System.currentTimeMillis() + expirationTime.toMillis();
cache.put(key, new CachedObject<>(value, expirationTime, expirationTimestamp));
cache.put(key, new CachedObject<>(value, expirationTime));
}

public void remove(K key) {
Expand Down
10 changes: 3 additions & 7 deletions ...ler-core/src/main/java/org/cloudfoundry/multiapps/controller/core/model/CachedObject.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,9 @@ public CachedObject(Duration expirationDuration) {
}

public CachedObject(T object, Duration expirationDuration) {
this(object, expirationDuration, System.currentTimeMillis() + expirationDuration.toMillis());
}

public CachedObject(T object, Duration expirationDuration, long expirationTimestamp) {
this.object = object;
this.expirationDuration = expirationDuration;
this.expirationTimestamp = expirationTimestamp;
this.expirationTimestamp = System.currentTimeMillis() + expirationDuration.toMillis();
}

public synchronized T get() {
Expand All @@ -30,14 +26,14 @@ public synchronized T get() {

public synchronized T getOrRefresh(Supplier<T> refresher) {
if (isExpired() || object == null) {
expirationTimestamp += expirationDuration.toMillis();
expirationTimestamp = System.currentTimeMillis() + expirationDuration.toMillis();
object = refresher.get();
}
return object;
}

public synchronized void refresh(Supplier<T> refresher) {
expirationTimestamp += expirationDuration.toMillis();
expirationTimestamp = System.currentTimeMillis() + expirationDuration.toMillis();
object = refresher.get();
}

Expand Down
6 changes: 3 additions & 3 deletions ...core/src/test/java/org/cloudfoundry/multiapps/controller/core/model/CachedObjectTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ void testCaching() {

@Test
void testExpiration() {
CachedObject<String> cache = new CachedObject<>("a", Duration.ofMillis(1), 2);
CachedObject<String> cache = new CachedObject<>("a", Duration.ofMillis(1));

assertFalse(cache.isExpired(1));
assertTrue(cache.isExpired(3));
assertFalse(cache.isExpired(System.currentTimeMillis() - 1000));
assertTrue(cache.isExpired(System.currentTimeMillis() + 3000));
}
}
18 changes: 0 additions & 18 deletions .../main/java/org/cloudfoundry/multiapps/controller/persistence/dto/DatabaseLockDetails.java
View file

This file was deleted.

0 comments on commit fff4ae4

Please sign in to comment.