Skip to content

Commit

Permalink
Adding more ssl user agent error stats (#11946)
Browse files Browse the repository at this point in the history 
Half of our production user agent ssl error stats were other_errors
stats. These new stats cover over 80% of the previously other_errors
stats in our production environment, adding more specific visibility
into client-side SSL errors.
  • Loading branch information
@bneradt
bneradt authored Jan 11, 2025
1 parent 774b315 commit 800bbf9
Show file tree
Hide file tree
Showing 4 changed files with 153 additions and 68 deletions.
44 changes: 44 additions & 0 deletions doc/admin-guide/monitoring/statistics/core/ssl.en.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,50 @@ SSL/TLS
Incoming client SSL connections which presented a client certificate that had
been revoked, since statistics collection began.

.. ts:stat:: global proxy.process.ssl.user_agent_decryption_failed_or_bad_record_mac integer
:type: counter

Incoming client SSL connections which failed decryption or had a mismatched
MAC, since statistics collection began.

.. ts:stat:: global proxy.process.ssl.user_agent_http_request integer
:type: counter

Incoming client SSL connections which attempted to use plaintext HTTP without
SSL encryption, since statistics collection began.

.. ts:stat:: global proxy.process.ssl.user_agent_inappropriate_fallback integer
:type: counter

Incoming client SSL connections which used a fallback to an older TLS version
that |TS| doesn't support, since statistics collection began.

.. ts:stat:: global proxy.process.ssl.user_agent_no_shared_cipher integer
:type: counter

Incoming client SSL connections which failed due to no match in supported
ciphers between the client and |TS|, since statistics collection began.

.. ts:stat:: global proxy.process.ssl.user_agent_version_too_high integer
:type: counter

Incoming client SSL connections which failed due to the client only
supporting TLS versions that are too high for |TS| to support, since
statistics collection began.

This stat is only incremented when |TS| is built against an SSL library, such
as OpenSSL, that supports the ``SSL_R_VERSION_TOO_HIGH`` error.

.. ts:stat:: global proxy.process.ssl.user_agent_version_too_low integer
:type: counter

Incoming client SSL connections which failed due to the client only
supporting TLS versions that are too low for |TS| to accept, since statistics
collection began.

This stat is only incremented when |TS| is built against an SSL library, such
as OpenSSL, that supports the ``SSL_R_VERSION_TOO_LOW`` error.

.. ts:stat:: global proxy.process.ssl.user_agent_session_hit integer
:type: counter

Expand Down
28 changes: 28 additions & 0 deletions src/iocore/net/SSLDiags.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,15 @@
#include "P_SSLNetVConnection.h"

static DbgCtl ssl_diags_dbg_ctl{"ssl-diag"};
static DbgCtl ssl_error_dbg_ctl{"ssl-error"};

// return true if we have a stat for the error
static bool
increment_ssl_client_error(unsigned long err)
{
// we only look for LIB_SSL errors atm
if (ERR_LIB_SSL != ERR_GET_LIB(err)) {
Dbg(ssl_error_dbg_ctl, "User agent SSL error not ERR_LIB_SSL: %lu", err);
Metrics::Counter::increment(ssl_rsb.user_agent_other_errors);
return false;
}
Expand Down Expand Up @@ -69,7 +71,31 @@ increment_ssl_client_error(unsigned long err)
case SSL_R_TLSV1_ALERT_UNKNOWN_CA:
Metrics::Counter::increment(ssl_rsb.user_agent_unknown_ca);
break;
case SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC:
Metrics::Counter::increment(ssl_rsb.user_agent_decryption_failed_or_bad_record_mac);
break;
case SSL_R_HTTP_REQUEST:
Metrics::Counter::increment(ssl_rsb.user_agent_http_request);
break;
case SSL_R_INAPPROPRIATE_FALLBACK:
Metrics::Counter::increment(ssl_rsb.user_agent_inappropriate_fallback);
break;
case SSL_R_NO_SHARED_CIPHER:
Metrics::Counter::increment(ssl_rsb.user_agent_no_shared_cipher);
break;
#ifdef SSL_R_VERSION_TOO_HIGH
case SSL_R_VERSION_TOO_HIGH:
Metrics::Counter::increment(ssl_rsb.user_agent_version_too_high);
break;
#endif /* SSL_R_VERSION_TOO_HIGH */
#ifdef SSL_R_VERSION_TOO_LOW
case SSL_R_VERSION_TOO_LOW:
Metrics::Counter::increment(ssl_rsb.user_agent_version_too_low);
break;
#endif /* SSL_R_VERSION_TOO_LOW */

default:
Dbg(ssl_error_dbg_ctl, "Unknown user agent SSL error: %d", ERR_GET_REASON(err));
Metrics::Counter::increment(ssl_rsb.user_agent_other_errors);
return false;
}
Expand All @@ -84,6 +110,7 @@ increment_ssl_server_error(unsigned long err)
{
// we only look for LIB_SSL errors atm
if (ERR_LIB_SSL != ERR_GET_LIB(err)) {
Dbg(ssl_error_dbg_ctl, "Origin server SSL error not ERR_LIB_SSL: %lu", err);
Metrics::Counter::increment(ssl_rsb.origin_server_other_errors);
return false;
}
Expand Down Expand Up @@ -117,6 +144,7 @@ increment_ssl_server_error(unsigned long err)
Metrics::Counter::increment(ssl_rsb.origin_server_unknown_ca);
break;
default:
Dbg(ssl_error_dbg_ctl, "Unknown origin server SSL error: %d", ERR_GET_REASON(err));
Metrics::Counter::increment(ssl_rsb.origin_server_other_errors);
return false;
}
Expand Down
27 changes: 17 additions & 10 deletions src/iocore/net/SSLStats.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,16 +209,23 @@ SSLInitializeStatistics()
ssl_rsb.user_agent_bad_cert = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_bad_cert");
ssl_rsb.user_agent_cert_verify_failed = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_cert_verify_failed");
ssl_rsb.user_agent_decryption_failed = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_decryption_failed");
ssl_rsb.user_agent_expired_cert = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_expired_cert");
ssl_rsb.user_agent_other_errors = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_other_errors");
ssl_rsb.user_agent_revoked_cert = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_revoked_cert");
ssl_rsb.user_agent_session_hit = Metrics::Gauge::createPtr("proxy.process.ssl.user_agent_session_hit");
ssl_rsb.user_agent_session_miss = Metrics::Gauge::createPtr("proxy.process.ssl.user_agent_session_miss");
ssl_rsb.user_agent_session_timeout = Metrics::Gauge::createPtr("proxy.process.ssl.user_agent_session_timeout");
ssl_rsb.user_agent_sessions = Metrics::Gauge::createPtr("proxy.process.ssl.user_agent_sessions");
ssl_rsb.user_agent_unknown_ca = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_unknown_ca");
ssl_rsb.user_agent_unknown_cert = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_unknown_cert");
ssl_rsb.user_agent_wrong_version = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_wrong_version");
ssl_rsb.user_agent_decryption_failed_or_bad_record_mac =
Metrics::Counter::createPtr("proxy.process.ssl.user_agent_decryption_failed_or_bad_record_mac");
ssl_rsb.user_agent_expired_cert = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_expired_cert");
ssl_rsb.user_agent_http_request = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_http_request");
ssl_rsb.user_agent_inappropriate_fallback = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_inappropriate_fallback");
ssl_rsb.user_agent_no_shared_cipher = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_no_shared_cipher");
ssl_rsb.user_agent_other_errors = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_other_errors");
ssl_rsb.user_agent_revoked_cert = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_revoked_cert");
ssl_rsb.user_agent_session_hit = Metrics::Gauge::createPtr("proxy.process.ssl.user_agent_session_hit");
ssl_rsb.user_agent_session_miss = Metrics::Gauge::createPtr("proxy.process.ssl.user_agent_session_miss");
ssl_rsb.user_agent_session_timeout = Metrics::Gauge::createPtr("proxy.process.ssl.user_agent_session_timeout");
ssl_rsb.user_agent_sessions = Metrics::Gauge::createPtr("proxy.process.ssl.user_agent_sessions");
ssl_rsb.user_agent_unknown_ca = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_unknown_ca");
ssl_rsb.user_agent_unknown_cert = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_unknown_cert");
ssl_rsb.user_agent_version_too_high = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_version_too_high");
ssl_rsb.user_agent_version_too_low = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_version_too_low");
ssl_rsb.user_agent_wrong_version = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_wrong_version");

#if defined(OPENSSL_IS_BORINGSSL)
size_t n = SSL_get_all_cipher_names(nullptr, 0);
Expand Down
122 changes: 64 additions & 58 deletions src/iocore/net/SSLStats.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,64 +39,70 @@ using ts::Metrics;
// for ssl_rsb.total_ticket_keys_renewed needs this initialization, but lets be
// consistent at least.
struct SSLStatsBlock {
Metrics::Counter::AtomicType *early_data_received_count = nullptr;
Metrics::Counter::AtomicType *error_async = nullptr;
Metrics::Counter::AtomicType *error_ssl = nullptr;
Metrics::Counter::AtomicType *error_syscall = nullptr;
Metrics::Counter::AtomicType *ocsp_refresh_cert_failure = nullptr;
Metrics::Counter::AtomicType *ocsp_refreshed_cert = nullptr;
Metrics::Counter::AtomicType *ocsp_revoked_cert = nullptr;
Metrics::Counter::AtomicType *ocsp_unknown_cert = nullptr;
Metrics::Counter::AtomicType *origin_server_bad_cert = nullptr;
Metrics::Counter::AtomicType *origin_server_cert_verify_failed = nullptr;
Metrics::Counter::AtomicType *origin_server_decryption_failed = nullptr;
Metrics::Counter::AtomicType *origin_server_expired_cert = nullptr;
Metrics::Counter::AtomicType *origin_server_other_errors = nullptr;
Metrics::Counter::AtomicType *origin_server_revoked_cert = nullptr;
Metrics::Counter::AtomicType *origin_server_unknown_ca = nullptr;
Metrics::Counter::AtomicType *origin_server_unknown_cert = nullptr;
Metrics::Counter::AtomicType *origin_server_wrong_version = nullptr;
Metrics::Counter::AtomicType *origin_session_cache_hit = nullptr;
Metrics::Counter::AtomicType *origin_session_cache_miss = nullptr;
Metrics::Counter::AtomicType *origin_session_reused_count = nullptr;
Metrics::Counter::AtomicType *session_cache_eviction = nullptr;
Metrics::Counter::AtomicType *session_cache_hit = nullptr;
Metrics::Counter::AtomicType *session_cache_lock_contention = nullptr;
Metrics::Counter::AtomicType *session_cache_miss = nullptr;
Metrics::Counter::AtomicType *session_cache_new_session = nullptr;
Metrics::Counter::AtomicType *sni_name_set_failure = nullptr;
Metrics::Counter::AtomicType *total_attempts_handshake_count_in = nullptr;
Metrics::Counter::AtomicType *total_attempts_handshake_count_out = nullptr;
Metrics::Counter::AtomicType *total_dyn_def_tls_record_count = nullptr;
Metrics::Counter::AtomicType *total_dyn_max_tls_record_count = nullptr;
Metrics::Counter::AtomicType *total_dyn_redo_tls_record_count = nullptr;
Metrics::Counter::AtomicType *total_handshake_time = nullptr;
Metrics::Counter::AtomicType *total_sslv3 = nullptr;
Metrics::Counter::AtomicType *total_success_handshake_count_in = nullptr;
Metrics::Counter::AtomicType *total_success_handshake_count_out = nullptr;
Metrics::Counter::AtomicType *total_ticket_keys_renewed = nullptr;
Metrics::Counter::AtomicType *total_tickets_created = nullptr;
Metrics::Counter::AtomicType *total_tickets_not_found = nullptr;
Metrics::Counter::AtomicType *total_tickets_renewed = nullptr;
Metrics::Counter::AtomicType *total_tickets_verified_old_key = nullptr;
Metrics::Counter::AtomicType *total_tickets_verified = nullptr;
Metrics::Counter::AtomicType *total_tlsv1 = nullptr;
Metrics::Counter::AtomicType *total_tlsv11 = nullptr;
Metrics::Counter::AtomicType *total_tlsv12 = nullptr;
Metrics::Counter::AtomicType *total_tlsv13 = nullptr;
Metrics::Counter::AtomicType *user_agent_bad_cert = nullptr;
Metrics::Counter::AtomicType *user_agent_cert_verify_failed = nullptr;
Metrics::Counter::AtomicType *user_agent_decryption_failed = nullptr;
Metrics::Counter::AtomicType *user_agent_expired_cert = nullptr;
Metrics::Counter::AtomicType *user_agent_other_errors = nullptr;
Metrics::Counter::AtomicType *user_agent_revoked_cert = nullptr;
Metrics::Gauge::AtomicType *user_agent_session_hit = nullptr;
Metrics::Gauge::AtomicType *user_agent_session_miss = nullptr;
Metrics::Gauge::AtomicType *user_agent_session_timeout = nullptr;
Metrics::Gauge::AtomicType *user_agent_sessions = nullptr;
Metrics::Counter::AtomicType *user_agent_unknown_ca = nullptr;
Metrics::Counter::AtomicType *user_agent_unknown_cert = nullptr;
Metrics::Counter::AtomicType *user_agent_wrong_version = nullptr;
Metrics::Counter::AtomicType *early_data_received_count = nullptr;
Metrics::Counter::AtomicType *error_async = nullptr;
Metrics::Counter::AtomicType *error_ssl = nullptr;
Metrics::Counter::AtomicType *error_syscall = nullptr;
Metrics::Counter::AtomicType *ocsp_refresh_cert_failure = nullptr;
Metrics::Counter::AtomicType *ocsp_refreshed_cert = nullptr;
Metrics::Counter::AtomicType *ocsp_revoked_cert = nullptr;
Metrics::Counter::AtomicType *ocsp_unknown_cert = nullptr;
Metrics::Counter::AtomicType *origin_server_bad_cert = nullptr;
Metrics::Counter::AtomicType *origin_server_cert_verify_failed = nullptr;
Metrics::Counter::AtomicType *origin_server_decryption_failed = nullptr;
Metrics::Counter::AtomicType *origin_server_expired_cert = nullptr;
Metrics::Counter::AtomicType *origin_server_other_errors = nullptr;
Metrics::Counter::AtomicType *origin_server_revoked_cert = nullptr;
Metrics::Counter::AtomicType *origin_server_unknown_ca = nullptr;
Metrics::Counter::AtomicType *origin_server_unknown_cert = nullptr;
Metrics::Counter::AtomicType *origin_server_wrong_version = nullptr;
Metrics::Counter::AtomicType *origin_session_cache_hit = nullptr;
Metrics::Counter::AtomicType *origin_session_cache_miss = nullptr;
Metrics::Counter::AtomicType *origin_session_reused_count = nullptr;
Metrics::Counter::AtomicType *session_cache_eviction = nullptr;
Metrics::Counter::AtomicType *session_cache_hit = nullptr;
Metrics::Counter::AtomicType *session_cache_lock_contention = nullptr;
Metrics::Counter::AtomicType *session_cache_miss = nullptr;
Metrics::Counter::AtomicType *session_cache_new_session = nullptr;
Metrics::Counter::AtomicType *sni_name_set_failure = nullptr;
Metrics::Counter::AtomicType *total_attempts_handshake_count_in = nullptr;
Metrics::Counter::AtomicType *total_attempts_handshake_count_out = nullptr;
Metrics::Counter::AtomicType *total_dyn_def_tls_record_count = nullptr;
Metrics::Counter::AtomicType *total_dyn_max_tls_record_count = nullptr;
Metrics::Counter::AtomicType *total_dyn_redo_tls_record_count = nullptr;
Metrics::Counter::AtomicType *total_handshake_time = nullptr;
Metrics::Counter::AtomicType *total_sslv3 = nullptr;
Metrics::Counter::AtomicType *total_success_handshake_count_in = nullptr;
Metrics::Counter::AtomicType *total_success_handshake_count_out = nullptr;
Metrics::Counter::AtomicType *total_ticket_keys_renewed = nullptr;
Metrics::Counter::AtomicType *total_tickets_created = nullptr;
Metrics::Counter::AtomicType *total_tickets_not_found = nullptr;
Metrics::Counter::AtomicType *total_tickets_renewed = nullptr;
Metrics::Counter::AtomicType *total_tickets_verified_old_key = nullptr;
Metrics::Counter::AtomicType *total_tickets_verified = nullptr;
Metrics::Counter::AtomicType *total_tlsv1 = nullptr;
Metrics::Counter::AtomicType *total_tlsv11 = nullptr;
Metrics::Counter::AtomicType *total_tlsv12 = nullptr;
Metrics::Counter::AtomicType *total_tlsv13 = nullptr;
Metrics::Counter::AtomicType *user_agent_bad_cert = nullptr;
Metrics::Counter::AtomicType *user_agent_cert_verify_failed = nullptr;
Metrics::Counter::AtomicType *user_agent_decryption_failed = nullptr;
Metrics::Counter::AtomicType *user_agent_decryption_failed_or_bad_record_mac = nullptr;
Metrics::Counter::AtomicType *user_agent_expired_cert = nullptr;
Metrics::Counter::AtomicType *user_agent_http_request = nullptr;
Metrics::Counter::AtomicType *user_agent_inappropriate_fallback = nullptr;
Metrics::Counter::AtomicType *user_agent_no_shared_cipher = nullptr;
Metrics::Counter::AtomicType *user_agent_other_errors = nullptr;
Metrics::Counter::AtomicType *user_agent_revoked_cert = nullptr;
Metrics::Counter::AtomicType *user_agent_unknown_ca = nullptr;
Metrics::Counter::AtomicType *user_agent_unknown_cert = nullptr;
Metrics::Counter::AtomicType *user_agent_version_too_high = nullptr;
Metrics::Counter::AtomicType *user_agent_version_too_low = nullptr;
Metrics::Counter::AtomicType *user_agent_wrong_version = nullptr;
Metrics::Gauge::AtomicType *user_agent_session_hit = nullptr;
Metrics::Gauge::AtomicType *user_agent_session_miss = nullptr;
Metrics::Gauge::AtomicType *user_agent_session_timeout = nullptr;
Metrics::Gauge::AtomicType *user_agent_sessions = nullptr;
};

extern SSLStatsBlock ssl_rsb;
Expand Down

0 comments on commit 800bbf9

Please sign in to comment.