Merge branch 'trunk' into dev

VERSION

@@ -1 +1 @@
-2.5.25
+2.5.26

doc/CHANGELOG.md

@@ -1,5 +1,14 @@
 # CHANGELOG for Mercury
 
+## Version 2.5.26
+
+* Added SOCKS v4 and v5 identification and metadata reporting.
+* Added `tls/2` and `quic/1` fingerprint definitions.
+* Added DNS SVCB parsing.
+* Fixed SMB special character escaping.
+* Adjusted classifier malware probability estimation logic to better handle the case where there are few labeled benign samples.
+* Minor additions to internal classes and functions.
+
 ## Version 2.5.25
 
 * Fingerprints are reported for Tofsee initial messages as `tofsee/generic`.

doc/sphinx/source/index.rst

@@ -62,6 +62,10 @@ Mercury Library Documentation
    :project: mercury
    :members:
 
+.. doxygenclass:: sequence
+   :project: mercury
+   :members:
+
 .. doxygengroup:: byteorder
    :project: mercury
 

src/libmerc/datum.h

@@ -1575,8 +1575,17 @@ class lookahead {
     datum tmp;
 public:
 
+    /// construct a lookahead<T> object by parsing the datum d.
+    ///
     lookahead(datum d) : value{d}, tmp{d} { }
 
+    /// construct a lookahead<T> object by parsing the datum d while
+    /// passing the parameter p of type P to the constructor of the T
+    /// object.
+    ///
+    template <typename P>
+    lookahead(datum d, P p) : value{d}, tmp{d, p} { }
+
     explicit operator bool() const { return tmp.is_not_null(); }
 
     datum advance() const { return tmp; }
@@ -1668,4 +1677,52 @@ class ignore {
     }
 };
 
+/// parses a sequence of objects of type `T` from a datum, when used in
+/// a range-based for loop.
+///
+/// \note Objects of type `T` must be constructible from a \ref datum
+/// reference.
+///
+/// The following example shows how to read four \ref
+/// encoded<uint16_t> objects from a buffer.
+///
+/// \code
+///     uint8_t buffer[] = {
+///         0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef
+///     };
+///     datum d{buffer, buffer + sizeof(buffer)};
+///     for (encoded<uint16_t> x : sequence<encoded<uint16_t>>{d}) {
+///         printf("%04x\n", x.value());
+///     }
+/// \endcode
+///
+template <typename T>
+class sequence {
+    datum tmp;
+    T value;
+
+    static_assert(std::is_constructible_v<T, datum &>, "T must be constructible from a datum reference");
+
+    struct iterator {
+        sequence *seq;
+
+        void operator++() { seq->value = T{seq->tmp}; }
+
+        T& operator* () { return seq->value; }
+
+        bool operator!= (const iterator &) const { return seq->tmp.is_not_null(); }
+
+    };
+
+public:
+
+    sequence(const datum &d) : tmp{d}, value{tmp} { }
+
+    iterator begin() { return { this }; }
+
+    iterator end() { return { nullptr }; }
+
+};
+
+
 #endif /* DATUM_H */

src/libmerc/dns.h

@@ -17,6 +17,7 @@
 #include "json_object.h"
 #include "util_obj.h"
 #include "match.h"
+#include "ech.hpp"
 
 /**
  * \file dns.h
@@ -459,6 +460,209 @@ class soa_rdata {
     }
 };
 
+
+// length_prefixed_string is a character string proceeded by a uint8_t
+// length field, which is used in DNS SVCB.
+//
+class length_prefixed_string {
+    encoded<uint8_t> length;
+    datum value;
+
+public:
+
+    length_prefixed_string(datum &d) :
+        length{d},
+        value{d, length.value()}
+    {}
+
+    bool is_valid() const { return value.is_not_null(); }
+
+    const datum &get_value() const { return value; }
+};
+
+// SVCB (Service Binding) RDATA Wire Format (following RFC 9460)
+//
+// The RDATA for the SVCB RR consists of:
+//
+//    * a 2-octet field for SvcPriority as an integer in network byte order.
+//
+//    * the uncompressed, fully qualified TargetName, represented as a
+//      sequence of length-prefixed labels per Section 3.1 of
+//      [RFC1035].
+//
+//    * the SvcParams, consuming the remainder of the record (so
+//      smaller than 65535 octets and constrained by the RDATA and DNS
+//      message sizes).
+//
+// When the list of SvcParams is non-empty, it contains a series of
+// SvcParamKey=SvcParamValue pairs, represented as:
+//
+//    * a 2-octet field containing the SvcParamKey as an integer in
+//      network byte order. (See Section 14.3.2 for the defined
+//      values.)
+//
+//    * a 2-octet field containing the length of the SvcParamValue as
+//      an integer between 0 and 65535 in network byte order.
+//
+//    * an octet string of this length whose contents are the
+//      SvcParamValue in a format determined by the SvcParamKey.
+//
+// The SvcParamKeys SHALL appear in increasing numeric order.
+//
+// The keys have the following names and numbers:
+///
+//    Number        Name            Meaning
+//    0             mandatory       Mandatory keys in this RR
+//    1             alpn            Additional supported protocols
+//    2             no-default-alpn No support for default protocol
+//    3             port            Port for alternative endpoint
+//    4             ipv4hint        IPv4 address hints
+//    5             ech             RESERVED (held for Encrypted ClientHello)
+//    6             ipv6hint        IPv6 address hints
+//    65280-65534   N/A             Reserved for Private Use
+//    65535 N/A                     Reserved ("Invalid key")
+
+class svc_params {
+    encoded<uint16_t> key;
+    encoded<uint16_t> length;
+    datum value;
+
+public:
+
+    svc_params(datum &d) :
+        key{d},
+        length{d},
+        value{d, length}
+    { }
+
+    void write_json(json_object &o) const {
+        if (value.is_null()) {
+            return;
+        }
+        switch(key) {
+        case 0:
+            write_mandatory(o);
+            break;
+        case 1:
+            write_alpn(o);
+            break;
+        case 2:
+            write_no_default_alpn(o);
+            break;
+        case 3:
+            write_port(o);
+            break;
+        case 4:
+            write_ipv4hint(o);
+            break;
+        case 5:
+            write_ech(o);
+            break;
+        case 6:
+            write_ipv6hint(o);
+            break;
+        case 65535:
+            write_invalid_key(o);
+            break;
+        default:
+            write_unknown(o);
+            ;
+        }
+    }
+
+    void write_mandatory(json_object &o) const {
+        o.print_key_hex("mandatory", value);
+    }
+    void write_alpn(json_object &o) const {
+        json_array a{o, "alpn"};
+        datum tmp{value};
+        while (tmp.is_not_empty()) {
+            if (lookahead<length_prefixed_string> string{tmp}) {
+                a.print_json_string(string.value.get_value());
+                tmp = string.advance();
+            }
+        }
+        a.close();
+    }
+    void write_no_default_alpn(json_object &o) const {
+        o.print_key_hex("no_default_alpn", value);                // should be empty
+    }
+    void write_port(json_object &o) const {
+        o.print_key_string("key", "port");
+        if (lookahead<encoded<uint16_t>> p{value}) {
+            o.print_key_uint16("value", p.value);
+        }
+    }
+    void write_ipv4hint(json_object &o) const {
+        json_array a{o, "ipv4hint"};
+        datum tmp{value};
+        while (tmp.is_not_empty()) {
+            ipv4_addr addr{tmp};
+            if (tmp.is_null()) {
+                break;
+            }
+            a.print_key(addr);
+        }
+        a.close();
+    }
+    void write_ech(json_object &o) const {
+        if (lookahead<ech_config> config{value}) {
+            config.value.write_json(o);
+        }
+    }
+    void write_ipv6hint(json_object &o) const {
+        json_array a{o, "ipv6hint"};
+        datum tmp{value};
+        while (tmp.is_not_empty()) {
+            ipv6_addr addr{tmp};
+            if (tmp.is_null()) {
+                break;
+            }
+            a.print_key(addr);
+        }
+        a.close();
+    }
+    void write_invalid_key(json_object &o) const {
+        o.print_key_hex("invalid_key", value);
+    }
+    void write_unknown(json_object &o) const {
+        o.print_key_hex("unknown", value);
+    }
+
+};
+
+class svcb_rdata {
+    encoded<uint16_t> svc_priority;
+    dns_name target_name;
+    datum svc_param_list;
+    bool valid;
+
+public:
+
+    svcb_rdata(datum &d, const datum &dns_body) :
+        svc_priority{d},
+        target_name{d, dns_body},
+        svc_param_list{d},
+        valid{d.is_not_null() && !target_name.is_null()}
+    { }
+
+    void write_json(json_object &o) const {
+        if (valid) {
+            o.print_key_uint("priority", svc_priority);
+            o.print_key_json_string("target_name", target_name.buffer, target_name.readable_length());
+            json_array param_list{o, "svc_params"};
+
+            for (svc_params &params : sequence<svc_params>{svc_param_list}) {
+                json_object p{param_list};
+                params.write_json(p);
+                p.close();
+            }
+
+            param_list.close();
+        }
+    }
+};
+
 struct dns_question_record {
     struct dns_name name;
     uint16_t rr_type;
@@ -714,6 +918,15 @@ struct dns_resource_record {
                 } else if ((dns_rr_type)question_record.rr_type == dns_rr_type::SOA) {
                     soa_rdata soa{tmp_rdata, body};
                     soa.write_json(rr);
+
+                } else if ((dns_rr_type)question_record.rr_type == dns_rr_type::HTTPS) {
+                    svcb_rdata svcb{tmp_rdata, body};
+                    svcb.write_json(rr);
+
+                } else {
+                    rr.print_key_uint("unknown_rr_type", question_record.rr_type);
+                    rr.print_key_hex("unknown_rr_value", tmp_rdata);
+
                 }
             } else {
                 rr.print_key_hex("rdata", tmp_rdata);