wip: openvpn observations

oonidata/src/oonidata/models/dataformats.py

@@ -367,23 +367,17 @@ class NetworkEvent(BaseModel):
     dial_id: Optional[int] = None
     conn_id: Optional[int] = None
 
-@add_slots
-@dataclass
-class OpenVPNConnectStatus(BaseModel):
-    success: bool
-    failure: Union[Failure, bool] = None
 
 @add_slots
 @dataclass
 class OpenVPNHandshake(BaseModel):
-    bootstrap_time: float
+    handshake_time: float
     endpoint: str
     ip: str # we might want to make this optional, and scrub in favor of ASN/prefix
     port: int
     transport: str
     provider: str
     openvpn_options: Optional[Dict[str, str]] = None
-    status: OpenVPNConnectStatus
     t0: float
     t: float
     tags: Optional[List[str]] = None

oonidata/src/oonidata/models/nettests/openvpn.py

@@ -1,7 +1,10 @@
 from dataclasses import dataclass
 from typing import List, Optional
-from oonidata.compat import add_slots
-from oonidata.models.dataformats import (
+
+from ..compat import add_slots
+
+from ..base import BaseModel
+from ..dataformats import (
     BaseTestKeys,
     Failure,
     TCPConnect,
@@ -14,12 +17,15 @@
 @add_slots
 @dataclass
 class OpenVPNTestKeys(BaseTestKeys):
-    failure: Failure = None
     success: Optional[bool] = False
+    failure: Failure = None
 
     network_events: Optional[List[OpenVPNNetworkEvent]] = None
-    openvpn_handshake: Optional[List[OpenVPNHandshake]] = None
     tcp_connect: Optional[List[TCPConnect]] = None
+    openvpn_handshake: Optional[List[OpenVPNHandshake]] = None
+
+    bootstrap_time: Optional[float] = None
+    tunnel: str = None
 
 
 @add_slots

oonidata/src/oonidata/models/observations.py

@@ -375,7 +375,7 @@ class HTTPMiddleboxObservation:
     table_index=("measurement_uid", "observation_id", "measurement_start_time"),
 )
 @dataclass
-class OpenVPNHandshakeObservation:
+class OpenVPNObservation:
     measurement_meta: MeasurementMeta
     probe_meta: ProbeMeta
     processing_meta: ProcessingMeta
@@ -401,8 +401,8 @@ class OpenVPNHandshakeObservation:
 
     # OpenVPN handshake observation
     openvpn_handshake_failure: Optional[Failure] = None
-    openvpn_handshake_success: Optional[bool] = None
     openvpn_handshake_t: Optional[float] = None
+    openvpn_handshake_t0: Optional[float] = None
 
     # timing info about the handshake packets
     openvpn_handshake_hr_client_t: Optional[float] = None

oonipipeline/src/oonipipeline/transforms/measurement_transformer.py

@@ -25,6 +25,8 @@
     NetworkEvent,
     TCPConnect,
     TLSHandshake,
+    OpenVPNHandshake,
+    OpenVPNNetworkEvent,
     maybe_binary_data_to_bytes,
 )
 from oonidata.models.nettests.base_measurement import BaseMeasurement
@@ -36,6 +38,7 @@
     TCPObservation,
     TLSObservation,
     WebObservation,
+    OpenVPNObservation,
 )
 from oonidata.datautils import (
     InvalidCertificateChain,
@@ -760,6 +763,63 @@ def make_measurement_meta(msmt: BaseMeasurement, bucket_date: str) -> Measuremen
         measurement_start_time=measurement_start_time,
     )
 
+def count_key_exchange_packets(network_events: List[OpenVPNNetworkEvent]) -> int:
+    """
+    return number of packets exchanged in the SENT_KEY state
+    """
+    n = 0
+    for evt in network_events:
+        if evt.stage == "SENT_KEY" and evt.operation.startswith("packet_"):
+            n+=1
+    return n
+
+def measurement_to_openvpn_observation(
+    msmt_meta: MeasurementMeta,
+    probe_meta: ProbeMeta,
+    netinfodb: NetinfoDB,
+    openvpn_h: OpenVPNHandshake,
+    tcp_connect: Optional[List[TCPConnect]],
+    network_events: Optional[List[OpenVPNNetworkEvent]],
+) -> OpenVPNObservation:
+
+    oo = OpenVPNObservation(
+        measurement_meta=msmt_meta,
+        probe_meta=probe_meta,
+        failure=normalize_failure(openvpn_h.failure),
+        timestamp=make_timestamp(msmt_meta.measurement_start_time, openvpn_h.t),
+        success=openvpn_h.failure == None,
+        protocol="openvpn",
+    )
+
+    if len(tcp_connect) != 0:
+        tcp = tcp_connect[0]
+        oo.tcp_success = tcp.status.success
+        oo.tcp_failure = tcp.status.failure
+        oo.tcp_t = tcp.t
+
+    oo.handshake_failure = openvpn_h.failure
+    oo.handshake_t = openvpn_h.t
+    oo.handshake_t0 = openvpn_h.t0
+
+    for evt in network_events:
+        if evt.packet is not None:
+            if evt.packet.opcode == "P_CONTROL_HARD_RESET_CLIENT_V2":
+                oo.openvpn_handshake_hr_client_t = evt.t
+            elif evt.packet.opcode == "P_CONTROL_HARD_RESET_SERVER_V2":
+                oo.openvpn_handshake_hr_server_t = evt.t
+            elif "client_hello" in evt.tags:
+                oo.openvpn_handshake_clt_hello_t = evt.t
+            elif "server_hello" in evt.tags:
+                oo.openvpn_handshake_clt_hello_t = evt.t
+            elif evt.operation == "state" and evt.stage == "GOT_KEY":
+                oo.openvpn_handshake_got_keys__t = evt.t
+            elif evt.operation == "state" and evt.stage == "GENERATED_KEYS":
+                oo.openvpn_handshake_gen_keys__t = evt.t
+
+    oo.openvpn_handshake_key_exchg_n = count_key_exchange_packets(network_events)
+
+    return oo
+
 
 class MeasurementTransformer:
     """
@@ -911,7 +971,7 @@ def consume_web_observations(
 
         It will attempt to map them via the transaction_id or ip:port tuple.
 
-        Any observation that cannot be mapped will be returned inside of it's
+        Any observation that cannot be mapped will be returned inside of its
         own WebObservation with all other columns set to None.
         """
         web_obs_list: List[WebObservation] = []
@@ -1008,5 +1068,38 @@ def consume_web_observations(
 
         return web_obs_list
 
+    def make_openvpn_observations(self,
+        tcp_connect: Optional[List[TCPConnect]],
+        openvpn_handshakes: Optional[List[OpenVPNHandshake]],
+        network_events: Optional[List[OpenVPNNetworkEvent]],
+        bootstrap_time: float,
+    ) -> List[OpenVPNObservation]:
+        """
+        Returns a list of OpenVPNObservations by mapping all related
+        TCPObservations, OpenVPNNetworkevents and OpenVPNHandshakes.
+        """
+        openvpn_obs_list: List[OpenVPNObservation] = []
+
+        for openvpn_handshake in openvpn_handshakes:
+            web_obs_list.append(
+                measurement_to_openvpn_observation(
+                    msmt_meta=self.measurement_meta,
+                    probe_meta=self.probe_meta,
+                    netinfodb=self.netinfodb,
+                    tcp_connect=tcp_connect,
+                    openvpn_handshake=openvpn_handshake,
+                    network_events=network_events,
+                )
+            )
+
+        # TODO: can factor out function with web_observation
+        for idx, obs in enumerate(openvpn_obs_list):
+            obs.observation_id = f"{obs.measurement_meta.measurement_uid}_{idx}"
+            obs.created_at = datetime.now(timezone.utc).replace(
+                microsecond=0, tzinfo=None
+            )
+
+        return openvpn_obs_list
+
     def make_observations(self, measurement):
         assert RuntimeError("make_observations is not implemented")

oonipipeline/src/oonipipeline/transforms/nettests/openvpn.py

@@ -0,0 +1,21 @@
+from typing import List, Tuple
+from oonidata.models.nettests import OpenVPN
+from oonidata.models.observations import OpenVPNObservation
+
+from ..measurement_transformer import MeasurementTransformer
+
+
+class OpenVPNTransformer(MeasurementTransformer):
+    def make_observations(self, msmt: OpenVPN) -> Tuple[List[OpenVPNObservation]]:
+        openvpn_obs_list = []
+        if not msmt.test_keys:
+            return (openvpn_obs_list,)
+
+        tcp_observations = self.make_tcp_observations(msmt.tcp_connect)
+
+        return self.make_openvpn_observations(
+            tcp_observations=tcp_observations,
+            openvpn_handshakes=msmt.openvpn_handshake,
+            network_events=msmt.network_events,
+            bootstrap_time=msmt.bootstrap_time,
+        )

oonipipeline/src/oonipipeline/transforms/observations.py

@@ -18,6 +18,7 @@
 from .nettests.browser_web import BrowserWebTransformer
 from .nettests.urlgetter import UrlGetterTransformer
 from .nettests.web_connectivity import WebConnectivityTransformer
+from .nettests.openvpn import OpenVPNTransformer 
 from .nettests.http_invalid_request_line import (
     HTTPInvalidRequestLineTransformer,
 )
@@ -37,6 +38,7 @@
     "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer,
     "http_invalid_request_line": HTTPInvalidRequestLineTransformer,
     "web_connectivity": WebConnectivityTransformer,
+    "openvpn": OpenVPNTransformer,
 }
 
 TypeWebConnectivityObservations = Tuple[