diff --git a/oonidata/src/oonidata/models/dataformats.py b/oonidata/src/oonidata/models/dataformats.py index 72ca8932..607de72e 100644 --- a/oonidata/src/oonidata/models/dataformats.py +++ b/oonidata/src/oonidata/models/dataformats.py @@ -367,23 +367,17 @@ class NetworkEvent(BaseModel): dial_id: Optional[int] = None conn_id: Optional[int] = None -@add_slots -@dataclass -class OpenVPNConnectStatus(BaseModel): - success: bool - failure: Union[Failure, bool] = None @add_slots @dataclass class OpenVPNHandshake(BaseModel): - bootstrap_time: float + handshake_time: float endpoint: str ip: str # we might want to make this optional, and scrub in favor of ASN/prefix port: int transport: str provider: str openvpn_options: Optional[Dict[str, str]] = None - status: OpenVPNConnectStatus t0: float t: float tags: Optional[List[str]] = None diff --git a/oonidata/src/oonidata/models/nettests/openvpn.py b/oonidata/src/oonidata/models/nettests/openvpn.py index eb1e0d3c..a8f3990e 100644 --- a/oonidata/src/oonidata/models/nettests/openvpn.py +++ b/oonidata/src/oonidata/models/nettests/openvpn.py @@ -1,7 +1,10 @@ from dataclasses import dataclass from typing import List, Optional -from oonidata.compat import add_slots -from oonidata.models.dataformats import ( + +from ..compat import add_slots + +from ..base import BaseModel +from ..dataformats import ( BaseTestKeys, Failure, TCPConnect, @@ -14,12 +17,15 @@ @add_slots @dataclass class OpenVPNTestKeys(BaseTestKeys): - failure: Failure = None success: Optional[bool] = False + failure: Failure = None network_events: Optional[List[OpenVPNNetworkEvent]] = None - openvpn_handshake: Optional[List[OpenVPNHandshake]] = None tcp_connect: Optional[List[TCPConnect]] = None + openvpn_handshake: Optional[List[OpenVPNHandshake]] = None + + bootstrap_time: Optional[float] = None + tunnel: str = None @add_slots diff --git a/oonidata/src/oonidata/models/observations.py b/oonidata/src/oonidata/models/observations.py index 85a6d3cf..aee6f12d 100644 --- a/oonidata/src/oonidata/models/observations.py +++ b/oonidata/src/oonidata/models/observations.py @@ -375,7 +375,7 @@ class HTTPMiddleboxObservation: table_index=("measurement_uid", "observation_id", "measurement_start_time"), ) @dataclass -class OpenVPNHandshakeObservation: +class OpenVPNObservation: measurement_meta: MeasurementMeta probe_meta: ProbeMeta processing_meta: ProcessingMeta @@ -401,8 +401,8 @@ class OpenVPNHandshakeObservation: # OpenVPN handshake observation openvpn_handshake_failure: Optional[Failure] = None - openvpn_handshake_success: Optional[bool] = None openvpn_handshake_t: Optional[float] = None + openvpn_handshake_t0: Optional[float] = None # timing info about the handshake packets openvpn_handshake_hr_client_t: Optional[float] = None diff --git a/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py b/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py index bad43404..ce93c735 100644 --- a/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py +++ b/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py @@ -25,6 +25,8 @@ NetworkEvent, TCPConnect, TLSHandshake, + OpenVPNHandshake, + OpenVPNNetworkEvent, maybe_binary_data_to_bytes, ) from oonidata.models.nettests.base_measurement import BaseMeasurement @@ -36,6 +38,7 @@ TCPObservation, TLSObservation, WebObservation, + OpenVPNObservation, ) from oonidata.datautils import ( InvalidCertificateChain, @@ -760,6 +763,63 @@ def make_measurement_meta(msmt: BaseMeasurement, bucket_date: str) -> Measuremen measurement_start_time=measurement_start_time, ) +def count_key_exchange_packets(network_events: List[OpenVPNNetworkEvent]) -> int: + """ + return number of packets exchanged in the SENT_KEY state + """ + n = 0 + for evt in network_events: + if evt.stage == "SENT_KEY" and evt.operation.startswith("packet_"): + n+=1 + return n + +def measurement_to_openvpn_observation( + msmt_meta: MeasurementMeta, + probe_meta: ProbeMeta, + netinfodb: NetinfoDB, + openvpn_h: OpenVPNHandshake, + tcp_connect: Optional[List[TCPConnect]], + network_events: Optional[List[OpenVPNNetworkEvent]], +) -> OpenVPNObservation: + + oo = OpenVPNObservation( + measurement_meta=msmt_meta, + probe_meta=probe_meta, + failure=normalize_failure(openvpn_h.failure), + timestamp=make_timestamp(msmt_meta.measurement_start_time, openvpn_h.t), + success=openvpn_h.failure == None, + protocol="openvpn", + ) + + if len(tcp_connect) != 0: + tcp = tcp_connect[0] + oo.tcp_success = tcp.status.success + oo.tcp_failure = tcp.status.failure + oo.tcp_t = tcp.t + + oo.handshake_failure = openvpn_h.failure + oo.handshake_t = openvpn_h.t + oo.handshake_t0 = openvpn_h.t0 + + for evt in network_events: + if evt.packet is not None: + if evt.packet.opcode == "P_CONTROL_HARD_RESET_CLIENT_V2": + oo.openvpn_handshake_hr_client_t = evt.t + elif evt.packet.opcode == "P_CONTROL_HARD_RESET_SERVER_V2": + oo.openvpn_handshake_hr_server_t = evt.t + elif "client_hello" in evt.tags: + oo.openvpn_handshake_clt_hello_t = evt.t + elif "server_hello" in evt.tags: + oo.openvpn_handshake_clt_hello_t = evt.t + elif evt.operation == "state" and evt.stage == "GOT_KEY": + oo.openvpn_handshake_got_keys__t = evt.t + elif evt.operation == "state" and evt.stage == "GENERATED_KEYS": + oo.openvpn_handshake_gen_keys__t = evt.t + + oo.openvpn_handshake_key_exchg_n = count_key_exchange_packets(network_events) + + return oo + class MeasurementTransformer: """ @@ -911,7 +971,7 @@ def consume_web_observations( It will attempt to map them via the transaction_id or ip:port tuple. - Any observation that cannot be mapped will be returned inside of it's + Any observation that cannot be mapped will be returned inside of its own WebObservation with all other columns set to None. """ web_obs_list: List[WebObservation] = [] @@ -1008,5 +1068,38 @@ def consume_web_observations( return web_obs_list + def make_openvpn_observations(self, + tcp_connect: Optional[List[TCPConnect]], + openvpn_handshakes: Optional[List[OpenVPNHandshake]], + network_events: Optional[List[OpenVPNNetworkEvent]], + bootstrap_time: float, + ) -> List[OpenVPNObservation]: + """ + Returns a list of OpenVPNObservations by mapping all related + TCPObservations, OpenVPNNetworkevents and OpenVPNHandshakes. + """ + openvpn_obs_list: List[OpenVPNObservation] = [] + + for openvpn_handshake in openvpn_handshakes: + web_obs_list.append( + measurement_to_openvpn_observation( + msmt_meta=self.measurement_meta, + probe_meta=self.probe_meta, + netinfodb=self.netinfodb, + tcp_connect=tcp_connect, + openvpn_handshake=openvpn_handshake, + network_events=network_events, + ) + ) + + # TODO: can factor out function with web_observation + for idx, obs in enumerate(openvpn_obs_list): + obs.observation_id = f"{obs.measurement_meta.measurement_uid}_{idx}" + obs.created_at = datetime.now(timezone.utc).replace( + microsecond=0, tzinfo=None + ) + + return openvpn_obs_list + def make_observations(self, measurement): assert RuntimeError("make_observations is not implemented") diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/openvpn.py b/oonipipeline/src/oonipipeline/transforms/nettests/openvpn.py new file mode 100644 index 00000000..2482f713 --- /dev/null +++ b/oonipipeline/src/oonipipeline/transforms/nettests/openvpn.py @@ -0,0 +1,21 @@ +from typing import List, Tuple +from oonidata.models.nettests import OpenVPN +from oonidata.models.observations import OpenVPNObservation + +from ..measurement_transformer import MeasurementTransformer + + +class OpenVPNTransformer(MeasurementTransformer): + def make_observations(self, msmt: OpenVPN) -> Tuple[List[OpenVPNObservation]]: + openvpn_obs_list = [] + if not msmt.test_keys: + return (openvpn_obs_list,) + + tcp_observations = self.make_tcp_observations(msmt.tcp_connect) + + return self.make_openvpn_observations( + tcp_observations=tcp_observations, + openvpn_handshakes=msmt.openvpn_handshake, + network_events=msmt.network_events, + bootstrap_time=msmt.bootstrap_time, + ) diff --git a/oonipipeline/src/oonipipeline/transforms/observations.py b/oonipipeline/src/oonipipeline/transforms/observations.py index f0ce3a2a..95e5c0a2 100644 --- a/oonipipeline/src/oonipipeline/transforms/observations.py +++ b/oonipipeline/src/oonipipeline/transforms/observations.py @@ -18,6 +18,7 @@ from .nettests.browser_web import BrowserWebTransformer from .nettests.urlgetter import UrlGetterTransformer from .nettests.web_connectivity import WebConnectivityTransformer +from .nettests.openvpn import OpenVPNTransformer from .nettests.http_invalid_request_line import ( HTTPInvalidRequestLineTransformer, ) @@ -37,6 +38,7 @@ "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer, "http_invalid_request_line": HTTPInvalidRequestLineTransformer, "web_connectivity": WebConnectivityTransformer, + "openvpn": OpenVPNTransformer, } TypeWebConnectivityObservations = Tuple[