CORE-20745: Modify UniRest client for SmokeTests (#6231)

- Tweak connection pooling parameters. Most importantly increase time to 60 seconds to allow more time for waiting for connection to become available in the pool.
- Log stats when we do run into situation when connection pool is depleted.

applications/workers/workers-smoketest/src/smokeTest/resources/commons-logging.properties

@@ -0,0 +1 @@
+org.apache.commons.logging.Log=org.apache.commons.logging.impl.SimpleLog

applications/workers/workers-smoketest/src/smokeTest/resources/simplelog.properties

@@ -0,0 +1,2 @@
+org.apache.commons.logging.simplelog.defaultlog=info
+org.apache.commons.logging.simplelog.showdatetime=true

testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/UnirestHttpsClient.kt

@@ -1,29 +1,61 @@
 package net.corda.e2etest.utilities
 
+import kong.unirest.Config
 import kong.unirest.Headers
+import kong.unirest.HttpRequestSummary
+import kong.unirest.HttpResponse
+import kong.unirest.Interceptor
 import kong.unirest.MultipartBody
 import kong.unirest.Unirest
 import kong.unirest.apache.ApacheClient
 import net.corda.tracing.addTraceContextToHttpRequest
-import org.apache.http.client.config.RequestConfig
 import org.apache.http.conn.ssl.NoopHostnameVerifier
 import org.apache.http.conn.ssl.TrustAllStrategy
 import org.apache.http.impl.client.HttpClients
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager
 import org.apache.http.ssl.SSLContexts
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.lang.reflect.Field
 import java.net.URI
 import java.net.http.HttpRequest
 import javax.net.ssl.SSLContext
 
-class UnirestHttpsClient(private val endpoint: URI, private val username: String, private val password: String)  :
+class UnirestHttpsClient(private val endpoint: URI, private val username: String, private val password: String) :
     HttpsClient {
+
     private companion object {
+        val logger: Logger = LoggerFactory.getLogger(UnirestHttpsClient::class.java)
+
         fun Headers.toInternal(): List<Pair<String, String>> {
             return all().map { it.name to it.value }
         }
-    }
 
-    init {
-        addSslParams()
+        init {
+            addSslParams()
+        }
+
+        private fun addSslParams() {
+            val sslContext: SSLContext = SSLContexts.custom()
+                .loadTrustMaterial(TrustAllStrategy())
+                .build()
+
+            val httpClient = HttpClients.custom()
+                .setSSLContext(sslContext)
+                .setSSLHostnameVerifier(NoopHostnameVerifier())
+                .setMaxConnPerRoute(3)
+                .setMaxConnTotal(21)
+                .build()
+
+            Unirest.config().let { config ->
+                config.interceptor(FailureReportingInterceptor(logger))
+                config.connectTimeout(60_000)
+                config.socketTimeout(60_000) // This parameter also controls `connectionRequestTimeout`.
+                    // I.e. duration of time to retrieve an available connection from the pool of connection
+                    // maintained by ApacheClient.
+                config.httpClient(ApacheClient.builder(httpClient).apply(config) as ApacheClient)
+            }
+        }
     }
 
     override fun postMultiPart(cmd: String, fields: Map<String, String>, files: Map<String, HttpsClientFileUpload>): SimpleResponse {
@@ -99,29 +131,6 @@ class UnirestHttpsClient(private val endpoint: URI, private val username: String
         return SimpleResponse(response.status, response.body, url, response.headers.toInternal())
     }
 
-    private fun addSslParams() {
-        val sslContext: SSLContext = SSLContexts.custom()
-            .loadTrustMaterial(TrustAllStrategy())
-            .build()
-
-        val requestConfig = RequestConfig.custom()
-            .setConnectionRequestTimeout(60_000)
-            .setConnectTimeout(60_000)
-            .setSocketTimeout(60_000)
-            .build()
-
-        val httpClient = HttpClients.custom()
-            .setSSLContext(sslContext)
-            .setSSLHostnameVerifier(NoopHostnameVerifier())
-            .setDefaultRequestConfig(requestConfig)
-            .build()
-
-        Unirest.config()
-            .let { config ->
-                config.httpClient(ApacheClient.builder(httpClient).apply(config))
-            }
-    }
-
     private fun MultipartBody.fields(fields: Map<String, String>): MultipartBody {
         fields.entries.forEach { (name, value) -> field(name, value) }
         return this
@@ -132,3 +141,40 @@ class UnirestHttpsClient(private val endpoint: URI, private val username: String
         return this
     }
 }
+
+private class FailureReportingInterceptor(private val logger: Logger) : Interceptor {
+    override fun onFail(e: Exception?, request: HttpRequestSummary?, config: Config?): HttpResponse<*> {
+
+        val clientFromConfig: ApacheClient? = config?.client as? ApacheClient
+
+        val connectionManager = clientFromConfig?.poolingConnectionManager
+        val statsString = if (connectionManager == null) {
+            "Connection manager is null"
+        } else {
+            val routesStats = connectionManager.routes.joinToString("\n") { route ->
+                "Route: $route => ${connectionManager.getStats(route)}"
+            }
+            "Total pool stats: ${connectionManager.totalStats}. Routes stats: $routesStats"
+        }
+
+        val configStr = config?.let {
+            "connectionTimeout=${it.connectionTimeout}, socketTimeout=${it.socketTimeout}"
+        } ?: ""
+
+        logger.error("Failed to process HTTP request (${request?.asString()}). $statsString. $configStr.", e)
+
+        return super.onFail(e, request, config)
+    }
+
+    private val ApacheClient.poolingConnectionManager: PoolingHttpClientConnectionManager?
+        get() {
+            if (manager != null) {
+                return manager
+            }
+
+            // Try obtaining from the private field
+            val maybeField: Result<Field> = runCatching { client.javaClass.getDeclaredField("connManager") }
+            return maybeField.getOrNull()?.also { it.isAccessible = true }
+                ?.get(client) as? PoolingHttpClientConnectionManager
+        }
+}