Add some heuristics to detect encrypted/obfuscated/proxied TLS flows

Based on the paper: "Fingerprinting Obfuscated Proxy Traffic with
Encapsulated TLS Handshakes".
See: https://www.usenix.org/conference/usenixsecurity24/presentation/xue-fingerprinting

Basic idea:
* the packets/bytes distribution of a TLS handshake is quite unique
* this fingerprint is still detectable if the handshake is
encrypted/proxied/obfuscated

All heuristics are disabled by default.

doc/configuration_parameters.md

@@ -25,6 +25,8 @@ TODO
 | NULL         | "lru.$CACHE_NAME.scope"                   | 0             | 0         | 1         | Set the scope of the specified LRU cache (0 = the cache is local, 1 = the cache is global). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, stun, tls_cert, mining, msteams, fpc_dns. The global scope con be set only if a global context has been initialized |
 | "tls"        | "certificate_expiration_threshold"        | 30            | 0         | 365       | The threshold (in days) used to trigger the `NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE` flow risk |
 | "tls"        | "application_blocks_tracking"             | disable       | NULL      | NULL      | Enable/disable processing of TLS Application Blocks (post handshake) to extract statistical information about the flow |
+| "tls  "      | "dpi.heuristics",                         | 0x00          | 0x00      | 0x07      | Enable/disable some heuristics to detect encrypted/obfuscated/proxied TLS flows. The value is a bitmask. Values: 0x0 = disabled; 0x01 = enable basic detection (i.e. encrypted TLS without any encapsulation); 0x02 = enable detection over TLS (i.e. TLS-in-TLS); 0x04 = enable detection over HTTP (i.e. TLS-over-WebSocket). If enabled, some false positives are expected. See: https://www.usenix.org/conference/usenixsecurity24/presentation/xue-fingerprinting |
+| "tls  "      | "dpi.heuristics.max_packets_extra_dissection",  | 25            | 0         | 255       | If at least one TLS heuristics is enabled (see `tls,"dpi.heuristics"`, this parameter set the upper limit on the number of packets required/processed for each flow. Higher the value, lower the false positive rate but more packets are required by nDPI for processing. |
 | "tls"        | "metadata.sha1_fingerprint"               | enable        | NULL      | NULL      | Enable/disable computation and export of SHA1 fingerprint for TLS flows. Note that if it is disable, the flow risk `NDPI_MALICIOUS_SHA1_CERTIFICATE` is not checked |
 | "tls"        | "metadata.ja3c_fingerprint"               | enable        | NULL      | NULL      | Enable/disable computation and export of JA3C fingerprint for TLS flows. Note that if it is disable, the flow risk `NDPI_MALICIOUS_JA3` is not checked |
 | "tls"        | "metadata.ja3s_fingerprint"               | enable        | NULL      | NULL      | Enable/disable computation and export of JA3S fingerprint for TLS flows |

example/ndpiReader.c

@@ -5946,6 +5946,25 @@ void memmemUnitTest(void) {
 
 /* *********************************************** */
 
+void mahalanobisUnitTest()
+{
+  /* Example based on: https://supplychenmanagement.com/2019/03/06/calculating-mahalanobis-distance/ */
+
+  const float i_s[3 * 3] = {  0.0482486100061447, -0.00420645518018837, -0.0138921893248235,
+                             -0.00420645518018836, 0.00177288408892603, -0.00649813703331057,
+                             -0.0138921893248235, -0.00649813703331056,  0.066800436339011 }; /* Inverted covar matrix */
+  const float u[3] = { 22.8, 180.0, 9.2 }; /* Means vector */
+  u_int32_t x[3] = { 26, 167, 12 }; /* Point */
+  float md;
+
+  md = ndpi_mahalanobis_distance(x, 3, u, i_s);
+  /* It is a bit tricky to test float equality on different archs -> loose check.
+   * md sholud be 1.3753 */
+  assert(md >= 1.37 && md <= 1.38);
+}
+
+/* *********************************************** */
+
 void filterUnitTest() {
   ndpi_filter* f = ndpi_filter_alloc();
   u_int32_t v, i;
@@ -6325,6 +6344,7 @@ int main(int argc, char **argv) {
     strnstrUnitTest();
     strncasestrUnitTest();
     memmemUnitTest();
+    mahalanobisUnitTest();
 #endif
   }
 

fuzz/fuzz_config.cpp

@@ -129,6 +129,14 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     ndpi_set_config(ndpi_info_mod, "tls", "application_blocks_tracking", cfg_value);
     ndpi_get_config(ndpi_info_mod, "tls", "application_blocks_tracking", cfg_value, sizeof(cfg_value));
   }
+  if(fuzzed_data.ConsumeBool()) {
+    value = fuzzed_data.ConsumeIntegralInRange(0, 0x07 + 1);
+    snprintf(cfg_value, sizeof(cfg_value), "%d", value);
+    ndpi_set_config(ndpi_info_mod, "tls", "dpi.heuristics", cfg_value);
+    value = fuzzed_data.ConsumeIntegralInRange(0, 255 + 1);
+    snprintf(cfg_value, sizeof(cfg_value), "%d", value);
+    ndpi_set_config(ndpi_info_mod, "tls", "dpi.heuristics.max_packets_extra_dissection", cfg_value);
+  }
   if(fuzzed_data.ConsumeBool()) {
     value = fuzzed_data.ConsumeIntegralInRange(0, 1 + 1);
     snprintf(cfg_value, sizeof(cfg_value), "%d", value);

fuzz/fuzz_ndpi_reader.c

@@ -88,6 +88,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     ndpi_set_config(workflow->ndpi_struct, "stun", "max_packets_extra_dissection", "255");
     ndpi_set_config(workflow->ndpi_struct, "zoom", "max_packets_extra_dissection", "255");
     ndpi_set_config(workflow->ndpi_struct, "rtp", "search_for_stun", "1");
+    ndpi_set_config(workflow->ndpi_struct, "tls", "dpi.heuristics", "0x07");
+    ndpi_set_config(workflow->ndpi_struct, "tls", "dpi.heuristics.max_packets_extra_dissection", "255");
 
     ndpi_finalize_initialization(workflow->ndpi_struct);
 

src/include/ndpi_api.h

@@ -1848,6 +1848,11 @@ extern "C" {
 
   /* ******************************* */
 
+  /* Mahalanobis distance (https://en.wikipedia.org/wiki/Mahalanobis_distance) between a point x and a distribution with mean u and inverted covariant matrix i_s */
+  float ndpi_mahalanobis_distance(const u_int32_t *x, u_int32_t size, const float *u, const float *i_s);
+
+  /* ******************************* */
+
   int  ndpi_init_bin(struct ndpi_bin *b, enum ndpi_bin_family f, u_int16_t num_bins);
   void ndpi_free_bin(struct ndpi_bin *b);
   struct ndpi_bin* ndpi_clone_bin(struct ndpi_bin *b);

src/include/ndpi_private.h

@@ -121,6 +121,7 @@ struct ndpi_packet_struct {
   struct ndpi_int_one_line_struct http_origin;
   struct ndpi_int_one_line_struct server_line;
   struct ndpi_int_one_line_struct http_method;
+  struct ndpi_int_one_line_struct upgrade_line;
   struct ndpi_int_one_line_struct http_response; /* the first "word" in this pointer is the
 						    response code in the packet (200, etc) */
 
@@ -232,6 +233,8 @@ struct ndpi_detection_module_config_struct {
 
   int tls_certificate_expire_in_x_days;
   int tls_app_blocks_tracking_enabled;
+  int tls_heuristics;
+  int tls_heuristics_max_packets;
   int tls_sha1_fingerprint_enabled;
   int tls_ja3c_fingerprint_enabled;
   int tls_ja3s_fingerprint_enabled;
@@ -609,6 +612,8 @@ u_int ndpi_search_tcp_or_udp_raw(struct ndpi_detection_module_struct *ndpi_struc
 
 char* ndpi_intoav4(unsigned int addr, char* buf, u_int16_t bufLen);
 
+int is_flow_addr_informative(const struct ndpi_flow_struct *flow);
+
 u_int16_t icmp4_checksum(u_int8_t const * const buf, size_t len);
 
 ndpi_risk_enum ndpi_network_risk_ptree_match(struct ndpi_detection_module_struct *ndpi_str,
@@ -637,7 +642,8 @@ void switch_to_tls(struct ndpi_detection_module_struct *ndpi_struct,
 int is_dtls(const u_int8_t *buf, u_int32_t buf_len, u_int32_t *block_len);
 void switch_extra_dissection_to_tls(struct ndpi_detection_module_struct *ndpi_struct,
 				    struct ndpi_flow_struct *flow);
-
+void switch_extra_dissection_to_tls_obfuscated_heur(struct ndpi_detection_module_struct* ndpi_struct,
+                                                    struct ndpi_flow_struct* flow);
 /* HTTP */
 void http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
                              struct ndpi_flow_struct *flow,

src/include/ndpi_typedefs.h

@@ -169,7 +169,8 @@ typedef enum {
   NDPI_MALWARE_HOST_CONTACTED, /* Flow client contacted a malware host */
   NDPI_BINARY_DATA_TRANSFER,   /* Attempt to transfer something in binary format */
   NDPI_PROBING_ATTEMPT,        /* Probing attempt (e.g. TCP connection with no data exchanged or unidirection traffic for bidirectional flows such as SSH) */
-
+  NDPI_OBFUSCATED_TRAFFIC,
+
   /* Leave this as last member */
   NDPI_MAX_RISK /* must be <= 63 due to (**) */
 } ndpi_risk_enum;
@@ -791,6 +792,13 @@ struct ndpi_lru_cache {
 /* Ookla */
 #define NDPI_AGGRESSIVENESS_OOKLA_TLS			0x01 /* Enable detection over TLS (using ookla cache) */
 
+/* TLS */
+#define NDPI_HEURISTICS_TLS_OBFUSCATED_PLAIN	0x01 /* Enable heuristic to detect proxied/obfuscated TLS flows over generic/unknown flows */
+#define NDPI_HEURISTICS_TLS_OBFUSCATED_TLS	0x02 /* Enable heuristic to detect proxied/obfuscated TLS flows over TLS tunnels, i.e. TLS over TLS */
+#define NDPI_HEURISTICS_TLS_OBFUSCATED_HTTP	0x04 /* Enable heuristic to detect proxied/obfuscated TLS flows over HTTP/WebSocket */
+
+
+
 /* ************************************************** */
 
 struct ndpi_flow_tcp_struct {
@@ -1298,6 +1306,7 @@ struct ndpi_flow_struct {
   struct {
     ndpi_http_method method;
     u_int8_t request_version; /* 0=1.0 and 1=1.1. Create an enum for this? */
+    u_int8_t websocket:1, _pad:7;
     u_int16_t response_status_code; /* 200, 404, etc. */
     char *url, *content_type /* response */, *request_content_type /* e.g. for POST */, *user_agent, *server;
     char *detected_os; /* Via HTTP/QUIC User-Agent */
@@ -1324,7 +1333,8 @@ struct ndpi_flow_struct {
 
   struct {
     message_t message[2]; /* Directions */
-    u_int8_t certificate_processed:1, _pad:7;
+    u_int8_t certificate_processed:1, change_cipher_from_client:1, change_cipher_from_server:1, from_opportunistic_tls:1, pad:4;
+    struct tls_obfuscated_heuristic_state *obfuscated_heur_state;
   } tls_quic; /* Used also by DTLS and POPS/IMAPS/SMTPS/FTPS */
 
   union {
@@ -1534,7 +1544,7 @@ struct ndpi_flow_struct {
   /* Flow payload */
   u_int16_t flow_payload_len;
   char *flow_payload;
-  
+
   /* 
      Leave this field below at the end
      The field below can be used by third
@@ -1548,8 +1558,8 @@ struct ndpi_flow_struct {
 _Static_assert(sizeof(((struct ndpi_flow_struct *)0)->protos) <= 264,
                "Size of the struct member protocols increased to more than 264 bytes, "
                "please check if this change is necessary.");
-_Static_assert(sizeof(struct ndpi_flow_struct) <= 1128,
-               "Size of the flow struct increased to more than 1120 bytes, "
+_Static_assert(sizeof(struct ndpi_flow_struct) <= 1136,
+               "Size of the flow struct increased to more than 1136 bytes, "
                "please check if this change is necessary.");
 #endif
 #endif

src/lib/ndpi_analyze.c

@@ -2078,3 +2078,45 @@ void ndpi_popcount_count(struct ndpi_popcount *h, const u_int8_t *buf, u_int32_t
 
   h->tot_bytes_count += buf_len;
 }
+
+/* ********************************************************************************* */
+/* ********************************************************************************* */
+
+
+/* It provides the Mahalanobis distance (https://en.wikipedia.org/wiki/Mahalanobis_distance)
+   between a point x and a distribution with mean u and inverted covariant matrix i_s.
+   Parameters:
+    x: input array (with dimension "size")
+    u: means array (with dimension "size")
+    i_s: inverted covariant matrix (with dimension "size" * "size")
+
+   Bottom line: distance = sqrt([x - u] * [i_s] * [x - u]^T)
+*/
+float ndpi_mahalanobis_distance(const u_int32_t *x, u_int32_t size, const float *u, const float *i_s)
+{
+  float md = 0;
+  float *diff; /* [x - u] */
+  float *tmp;  /* Result of [x - u] * [i_s] */
+  u_int32_t i, j;
+
+  /* Could we get rid of these allocations? */
+  diff = ndpi_calloc(sizeof(float), size);
+  tmp = ndpi_calloc(sizeof(float), size);
+  if(diff && tmp) {
+    for (i = 0; i < size; i++)
+      diff[i] = x[i] - u[i];
+
+    /* Naive implementation of matrix multiplication(s) */
+    for(i = 0; i < size; i++) {
+      for(j = 0; j < size; j++) {
+        tmp[i] += diff[j] * i_s[size * j + i];
+      }
+    }
+    for(i = 0; i < size; i++)
+      md += tmp[i] * diff[i];
+  }
+  ndpi_free(diff);
+  ndpi_free(tmp);
+
+  return sqrt(md);
+}

src/lib/ndpi_main.c

@@ -198,6 +198,7 @@ static ndpi_risk_info ndpi_known_risks[] = {
   { NDPI_MALWARE_HOST_CONTACTED,                NDPI_RISK_SEVERE, CLIENT_HIGH_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
   { NDPI_BINARY_DATA_TRANSFER,                  NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
   { NDPI_PROBING_ATTEMPT,                       NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
+  { NDPI_OBFUSCATED_TRAFFIC,                    NDPI_RISK_HIGH,   CLIENT_HIGH_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
 
   /* Leave this as last member */
   { NDPI_MAX_RISK,                              NDPI_RISK_LOW,    CLIENT_FAIR_RISK_PERCENTAGE, NDPI_NO_ACCOUNTABILITY   }
@@ -438,6 +439,34 @@ void ndpi_set_proto_category(struct ndpi_detection_module_struct *ndpi_str, u_in
 
 /* ********************************************************************************** */
 
+int is_flow_addr_informative(const struct ndpi_flow_struct *flow)
+{
+  /* The ideas is to tell if the address itself carries some useful information or not.
+     Examples:
+      a flow to a Facebook address is quite likely related to some Facebook apps
+      a flow to an AWS address might be potentially anything
+  */
+
+  switch(flow->guessed_protocol_id_by_ip) {
+  case NDPI_PROTOCOL_UNKNOWN:
+  /* This is basically the list of cloud providers supported by nDPI */
+  case NDPI_PROTOCOL_TENCENT:
+  case NDPI_PROTOCOL_EDGECAST:
+  case NDPI_PROTOCOL_ALIBABA:
+  case NDPI_PROTOCOL_YANDEX_CLOUD:
+  case NDPI_PROTOCOL_AMAZON_AWS:
+  case NDPI_PROTOCOL_MICROSOFT_AZURE:
+  case NDPI_PROTOCOL_CACHEFLY:
+  case NDPI_PROTOCOL_CLOUDFLARE:
+  case NDPI_PROTOCOL_GOOGLE_CLOUD:
+    return 0;
+  default:
+    return 1;
+  }
+}
+
+/* ********************************************************************************** */
+
 /*
   There are some (master) protocols that are informative, meaning that it shows
   what is the subprotocol about, but also that the subprotocol isn't a real protocol.
@@ -6736,6 +6765,9 @@ void ndpi_free_flow_data(struct ndpi_flow_struct* flow) {
 
     if(flow->flow_payload != NULL)
       ndpi_free(flow->flow_payload);
+
+    if(flow->tls_quic.obfuscated_heur_state)
+      ndpi_free(flow->tls_quic.obfuscated_heur_state);
   }
 }
 
@@ -8270,6 +8302,7 @@ static void ndpi_reset_packet_line_info(struct ndpi_packet_struct *packet) {
     packet->server_line.len = 0, packet->http_method.ptr = NULL, packet->http_method.len = 0,
     packet->http_response.ptr = NULL, packet->http_response.len = 0,
     packet->forwarded_line.ptr = NULL, packet->forwarded_line.len = 0;
+    packet->upgrade_line.ptr = NULL, packet->upgrade_line.len = 0;
 }
 
 /* ********************************************************************************* */
@@ -8976,6 +9009,7 @@ static void parse_single_packet_line(struct ndpi_detection_module_struct *ndpi_s
                                      { "Authorization:", &packet->authorization_line },
                                      { NULL, NULL} };
   struct header_line headers_u[] = { { "User-agent:", &packet->user_agent_line },
+                                     { "Upgrade:", &packet->upgrade_line },
                                      { NULL, NULL} };
   struct header_line headers_c[] = { { "Content-Disposition:", &packet->content_disposition_line },
                                      { "Content-type:", &packet->content_line },
@@ -11402,6 +11436,8 @@ static const struct cfg_param {
 
   { "tls",           "certificate_expiration_threshold",        "30", "0", "365", CFG_PARAM_INT, __OFF(tls_certificate_expire_in_x_days), NULL },
   { "tls",           "application_blocks_tracking",             "disable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tls_app_blocks_tracking_enabled), NULL },
+  { "tls",           "dpi.heuristics",                          "0x00", "0", "0x07", CFG_PARAM_INT, __OFF(tls_heuristics), NULL },
+  { "tls",           "dpi.heuristics.max_packets_extra_dissection", "25", "0", "255", CFG_PARAM_INT, __OFF(tls_heuristics_max_packets), NULL },
   { "tls",           "metadata.sha1_fingerprint",               "enable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tls_sha1_fingerprint_enabled), NULL },
   { "tls",           "metadata.ja3c_fingerprint",               "enable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tls_ja3c_fingerprint_enabled), NULL },
   { "tls",           "metadata.ja3s_fingerprint",               "enable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tls_ja3s_fingerprint_enabled), NULL },

src/lib/ndpi_utils.c

@@ -2097,6 +2097,9 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) {
   case NDPI_PROBING_ATTEMPT:
     return("Probing Attempt");
 
+  case NDPI_OBFUSCATED_TRAFFIC:
+    return("Obfuscated Traffic");
+
   default:
     ndpi_snprintf(buf, sizeof(buf), "%d", (int)risk);
     return(buf);
@@ -2221,6 +2224,8 @@ const char* ndpi_risk2code(ndpi_risk_enum risk) {
     return STRINGIFY(NDPI_BINARY_DATA_TRANSFER);
   case NDPI_PROBING_ATTEMPT:
     return STRINGIFY(NDPI_PROBING_ATTEMPT);
+  case NDPI_OBFUSCATED_TRAFFIC:
+    return STRINGIFY(NDPI_OBFUSCATED_TRAFFIC);
 
   default:
     return("Unknown risk");
@@ -2342,6 +2347,8 @@ ndpi_risk_enum ndpi_code2risk(const char* risk) {
     return(NDPI_BINARY_DATA_TRANSFER);
   else if(strcmp(STRINGIFY(NDPI_PROBING_ATTEMPT), risk) == 0)
     return(NDPI_PROBING_ATTEMPT);
+  else if(strcmp(STRINGIFY(NDPI_OBFUSCATED_TRAFFIC), risk) == 0)
+    return(NDPI_OBFUSCATED_TRAFFIC);
   else
     return(NDPI_MAX_RISK);
 }

src/lib/protocols/http.c

@@ -148,7 +148,19 @@ static int ndpi_search_http_tcp_again(struct ndpi_detection_module_struct *ndpi_
 #endif
 
   if(flow->extra_packets_func == NULL) {
-    return(0); /* We're good now */
+    /* HTTP stuff completed */
+
+    /* Loook for TLS over websocket */
+    if((ndpi_struct->cfg.tls_heuristics & NDPI_HEURISTICS_TLS_OBFUSCATED_HTTP) && /* Feature enabled */
+       (flow->host_server_name[0] != '\0' &&
+        flow->http.response_status_code != 0) && /* Bidirectional HTTP traffic */
+       flow->http.websocket) {
+
+      switch_extra_dissection_to_tls_obfuscated_heur(ndpi_struct, flow);
+      return(1);
+    }
+
+    return(0); /* We are good now */
   }
 
   /* Possibly more processing */
@@ -954,6 +966,12 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
     }
   }
 
+  if(packet->upgrade_line.ptr != NULL) {
+    if(flow->http.response_status_code == 101 &&
+       memcmp((char *)packet->upgrade_line.ptr, "websocket", 9) == 0)
+      flow->http.websocket = 1;
+  }
+
   if(packet->server_line.ptr != NULL) {
     if(flow->http.server == NULL) {
       len = packet->server_line.len + 1;
@@ -1577,7 +1595,7 @@ static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struc
   ndpi_check_http_tcp(ndpi_struct, flow);
 
   if((ndpi_struct->cfg.http_parse_response_enabled &&
-      flow->host_server_name[0] != '\0'&&
+      flow->host_server_name[0] != '\0' &&
       flow->http.response_status_code != 0) ||
      (!ndpi_struct->cfg.http_parse_response_enabled &&
       (flow->host_server_name[0] != '\0' ||