diff --git a/.gitignore b/.gitignore index f2eac516..e660e491 100644 --- a/.gitignore +++ b/.gitignore @@ -148,6 +148,10 @@ doc/_build .luarocks +# vim +.*.swp +.*.swo + # vagrant .vagrant diff --git a/doc/design/establish_connection/sequence/base.puml b/doc/design/establish_connection/sequence/base.puml new file mode 100644 index 00000000..9f74961c --- /dev/null +++ b/doc/design/establish_connection/sequence/base.puml @@ -0,0 +1,15 @@ +@startuml + +!include ../../legend.puml + +box "Peer1" + participant "Frontend" as Peer1Frontend + participant "Backend" as Peer1Backend +end box + +box "Peer2" + participant "Backend" as Peer2Backend + participant "Frontend" as Peer2Frontend +end box + +@enduml \ No newline at end of file diff --git a/doc/design/establish_connection/sequence/both_peers_receive_register_peer.png b/doc/design/establish_connection/sequence/both_peers_receive_register_peer.png new file mode 100644 index 00000000..6d2de952 Binary files /dev/null and b/doc/design/establish_connection/sequence/both_peers_receive_register_peer.png differ diff --git a/doc/design/establish_connection/sequence/both_peers_receive_register_peer.puml b/doc/design/establish_connection/sequence/both_peers_receive_register_peer.puml new file mode 100644 index 00000000..a59067a6 --- /dev/null +++ b/doc/design/establish_connection/sequence/both_peers_receive_register_peer.puml @@ -0,0 +1,25 @@ +@startuml + +'https://plantuml.com/sequence-diagram + +!include base.puml + +par + [-[$method_call]> Peer1Frontend: register_peer() + Peer1Frontend -[$zmq_inproc]>> Peer1Backend: RegisterPeerMessage + Peer1Backend -[$zmq_tcp_rd_no_con]>> Peer2Backend: SynchronizeConnectionMessage +else + Peer2Frontend <[$method_call]-] : register_peer() + Peer2Backend <<[$zmq_inproc]- Peer2Frontend: RegisterPeerMessage + Peer1Backend <<[$zmq_tcp_rd_no_con]- Peer2Backend : SynchronizeConnectionMessage +end + +par + Peer1Backend <<[$zmq_tcp_rd_no_con]- Peer2Backend: AcknowledgeConnectionMessage + Peer1Backend -[$zmq_inproc]>> Peer1Frontend: PeerIsReadyMessage(Peer2) +else + Peer1Backend -[$zmq_tcp_rd_no_con]>> Peer2Backend: AcknowledgeConnectionMessage + Peer2Backend -[$zmq_inproc]>> Peer2Frontend: PeerIsReadyMessage(Peer1) +end + +@enduml diff --git a/doc/design/establish_connection/sequence/both_peers_receive_register_peer_both_peers_lose_synchonize.png b/doc/design/establish_connection/sequence/both_peers_receive_register_peer_both_peers_lose_synchonize.png new file mode 100644 index 00000000..de0c40a8 Binary files /dev/null and b/doc/design/establish_connection/sequence/both_peers_receive_register_peer_both_peers_lose_synchonize.png differ diff --git a/doc/design/establish_connection/sequence/both_peers_receive_register_peer_both_peers_lose_synchonize.puml b/doc/design/establish_connection/sequence/both_peers_receive_register_peer_both_peers_lose_synchonize.puml new file mode 100644 index 00000000..ae093cd4 --- /dev/null +++ b/doc/design/establish_connection/sequence/both_peers_receive_register_peer_both_peers_lose_synchonize.puml @@ -0,0 +1,32 @@ +@startuml +'https://plantuml.com/sequence-diagram + +!include base.puml + +par + [-[$method_call]> Peer1Frontend: register_peer() + Peer1Frontend -[$zmq_inproc]>> Peer1Backend: RegisterPeerMessage + Peer1Backend -[$zmq_tcp_rd_no_con]x Peer2Backend: SynchronizeConnectionMessage +else + Peer2Frontend <[$method_call]-] : register_peer() + Peer2Backend <<[$zmq_inproc]- Peer2Frontend: RegisterPeerMessage + Peer1Backend x[$zmq_tcp_rd_no_con]- Peer2Backend : SynchronizeConnectionMessage +end + +loop until abort timeout + par + ...synchronize_timeout... + Peer1Backend -[$zmq_tcp_rd_no_con]x Peer2Backend: SynchronizeConnectionMessage + else + ...synchronize_timeout... + Peer1Backend x[$zmq_tcp_rd_no_con]- Peer2Backend : SynchronizeConnectionMessage + end +end + +par + Peer2Backend -[$zmq_inproc]>> Peer2Frontend: TimeoutMessage +else + Peer1Frontend <<[$zmq_inproc]- Peer1Backend: TimeoutMessage +end + +@enduml \ No newline at end of file diff --git a/doc/design/establish_connection/sequence/both_peers_receive_register_peer_one_peer_loses_synchonize.png b/doc/design/establish_connection/sequence/both_peers_receive_register_peer_one_peer_loses_synchonize.png new file mode 100644 index 00000000..5030bb80 Binary files /dev/null and b/doc/design/establish_connection/sequence/both_peers_receive_register_peer_one_peer_loses_synchonize.png differ diff --git a/doc/design/establish_connection/sequence/both_peers_receive_register_peer_one_peer_loses_synchonize.puml b/doc/design/establish_connection/sequence/both_peers_receive_register_peer_one_peer_loses_synchonize.puml new file mode 100644 index 00000000..114e64de --- /dev/null +++ b/doc/design/establish_connection/sequence/both_peers_receive_register_peer_one_peer_loses_synchonize.puml @@ -0,0 +1,28 @@ +@startuml +'https://plantuml.com/sequence-diagram + +!include base.puml + +par + [-[$method_call]> Peer1Frontend: register_peer() + Peer1Frontend -[$zmq_inproc]>> Peer1Backend: RegisterPeerMessage + Peer1Backend -[$zmq_tcp_rd_no_con]>> Peer2Backend: SynchronizeConnectionMessage +else + Peer2Frontend <[$method_call]-] : register_peer() + Peer2Backend <<[$zmq_inproc]- Peer2Frontend: RegisterPeerMessage + Peer1Backend x[$zmq_tcp_rd_no_con]- Peer2Backend : SynchronizeConnectionMessage +end + +par + Peer1Backend <<[$zmq_tcp_rd_no_con]- Peer2Backend: AcknowledgeConnectionMessage + Peer1Backend -[$zmq_inproc]>> Peer1Frontend: PeerIsReadyMessage(Peer2) +else + loop until peer_is_ready_wait_time + ...synchronize_timeout... + Peer1Backend x[$zmq_tcp_rd_no_con]- Peer2Backend : SynchronizeConnectionMessage + end +end + +Peer2Backend -[$zmq_inproc]>> Peer2Frontend: PeerIsReadyMessage(Peer1) + +@enduml \ No newline at end of file diff --git a/doc/design/establish_connection/sequence/one_peer_receives_register_peer.png b/doc/design/establish_connection/sequence/one_peer_receives_register_peer.png new file mode 100644 index 00000000..2d51109d Binary files /dev/null and b/doc/design/establish_connection/sequence/one_peer_receives_register_peer.png differ diff --git a/doc/design/establish_connection/sequence/one_peer_receives_register_peer.puml b/doc/design/establish_connection/sequence/one_peer_receives_register_peer.puml new file mode 100644 index 00000000..a5c09f04 --- /dev/null +++ b/doc/design/establish_connection/sequence/one_peer_receives_register_peer.puml @@ -0,0 +1,18 @@ +@startuml +'https://plantuml.com/sequence-diagram + +!include base.puml + +[-[$method_call]> Peer1Frontend: register_peer() +Peer1Frontend -[$zmq_inproc]>> Peer1Backend: RegisterPeerMessage +Peer1Backend -[$zmq_tcp_rd_no_con]>> Peer2Backend: SynchronizeConnectionMessage +Peer1Backend <<[$zmq_tcp_rd_no_con]- Peer2Backend : SynchronizeConnectionMessage +par + Peer1Backend -[$zmq_tcp_rd_no_con]>> Peer2Backend: AcknowledgeConnectionMessage + Peer2Backend -[$zmq_inproc]>> Peer2Frontend: PeerIsReadyMessage(Peer1) +else + Peer1Backend <<[$zmq_tcp_rd_no_con]- Peer2Backend: AcknowledgeConnectionMessage + Peer1Frontend <<[$zmq_inproc]- Peer1Backend: PeerIsReadyMessage(Peer2) +end + +@enduml \ No newline at end of file diff --git a/doc/design/establish_connection/state_machine.png b/doc/design/establish_connection/state_machine.png new file mode 100644 index 00000000..f9e87b25 Binary files /dev/null and b/doc/design/establish_connection/state_machine.png differ diff --git a/doc/design/establish_connection/state_machine.puml b/doc/design/establish_connection/state_machine.puml new file mode 100644 index 00000000..9cfb4189 --- /dev/null +++ b/doc/design/establish_connection/state_machine.puml @@ -0,0 +1,18 @@ +@startuml +'https://plantuml.com/state-diagram + +[*] --> PeerIsRegistered : received RegisterPeerMessage / \n send SynchronizeConnectionMessage +[*] --> ReceivedSynchronizeConnectionMessage : received SynchronizeConnectionMessage / \n send SynchronizeConnectionMessage \n send AcknowledgeConnectionMessage +PeerIsRegistered --> ReceivedSynchronizeConnectionMessage : received SynchronizeConnectionMessage \n send AcknowledgeConnectionMessage +PeerIsRegistered --> ReceivedAcknowledgeConnectionMessage : received AcknowledgeConnectionMessage + +ReceivedSynchronizeConnectionMessage --> PeerIsReady : received AcknowledgeConnectionMessage + +PeerIsRegistered -> PeerIsRegistered : Timeout / \n send SynchronizeConnectionMessage + +ReceivedSynchronizeConnectionMessage --> ReceivedSynchronizeConnectionMessage : Timeout / \n send SynchronizeConnectionMessage +ReceivedSynchronizeConnectionMessage --> PeerIsReady : Waited a while + +ReceivedAcknowledgeConnectionMessage --> PeerIsReady : received SynchronizeConnectionMessage \n send AcknowledgeConnectionMessage +ReceivedAcknowledgeConnectionMessage --> PeerIsReady : Waited a while +@enduml \ No newline at end of file diff --git a/doc/design/legend.puml b/doc/design/legend.puml new file mode 100644 index 00000000..3ed95b6f --- /dev/null +++ b/doc/design/legend.puml @@ -0,0 +1,14 @@ +@startuml +!$udp = '#red' +!$method_call = '#green' +!$zmq_inproc = '#blue' +!$zmq_tcp_rd_no_con = '#darkviolet' + +legend + |Color| Protocol | + |<$udp>| UDP | + |<$method_call>| method calls | + |<$zmq_inproc>| ZMQ inproc | + |<$zmq_tcp_rd_no_con>| ZMQ TCP Dealer/Router Connection Less | +end legend +@enduml \ No newline at end of file diff --git a/doc/design/local_discovery_strategy/both_peer_receive_ping.png b/doc/design/local_discovery_strategy/both_peer_receive_ping.png new file mode 100644 index 00000000..fc193faa Binary files /dev/null and b/doc/design/local_discovery_strategy/both_peer_receive_ping.png differ diff --git a/doc/design/local_discovery_strategy/both_peer_receive_ping.puml b/doc/design/local_discovery_strategy/both_peer_receive_ping.puml new file mode 100644 index 00000000..f3731d9d --- /dev/null +++ b/doc/design/local_discovery_strategy/both_peer_receive_ping.puml @@ -0,0 +1,36 @@ +@startuml +'https://plantuml.com/sequence-diagram + +!include ../legend.puml + +box "Peer1" + participant "Frontend" as Peer1Frontend + participant "DiscoveryStrategy" as Peer1DiscoveryStrategy +end box + +box "Peer2" + participant "DiscoveryStrategy" as Peer2DiscoveryStrategy + participant "Frontend" as Peer2Frontend +end box + +par + loop are_all_peers_connected() + Peer1DiscoveryStrategy -[$udp]>> Peer2DiscoveryStrategy: PingMessage + Peer2DiscoveryStrategy -[$method_call]> Peer2Frontend: register_peer() + return + ref over Peer1Frontend, Peer2Frontend: Establish Connection + Peer2DiscoveryStrategy -[$method_call]> Peer2Frontend: are_all_peers_connected() + return + end +else + loop are_all_peers_connected() + Peer1DiscoveryStrategy <<[$udp]- Peer2DiscoveryStrategy: PingMessage + Peer1DiscoveryStrategy -[$method_call]> Peer1Frontend: register_peer() + return + ref over Peer1Frontend, Peer2Frontend: Establish Connection + Peer1DiscoveryStrategy -[$method_call]> Peer1Frontend: are_all_peers_connected() + return + end +end + +@enduml \ No newline at end of file diff --git a/doc/design/local_discovery_strategy/one_peer_receives_ping.png b/doc/design/local_discovery_strategy/one_peer_receives_ping.png new file mode 100644 index 00000000..ef19312a Binary files /dev/null and b/doc/design/local_discovery_strategy/one_peer_receives_ping.png differ diff --git a/doc/design/local_discovery_strategy/one_peer_receives_ping.puml b/doc/design/local_discovery_strategy/one_peer_receives_ping.puml new file mode 100644 index 00000000..031de52d --- /dev/null +++ b/doc/design/local_discovery_strategy/one_peer_receives_ping.puml @@ -0,0 +1,31 @@ +@startuml +'https://plantuml.com/sequence-diagram + +!include ../legend.puml + +box "Peer1" + participant "Frontend" as Peer1Frontend + participant "DiscoveryStrategy" as Peer1DiscoveryStrategy +end box + +box "Peer2" + participant "DiscoveryStrategy" as Peer2DiscoveryStrategy + participant "Frontend" as Peer2Frontend +end box + +par + loop are_all_peers_connected() + Peer1DiscoveryStrategy -[$udp]>> Peer2DiscoveryStrategy: PingMessage + Peer2DiscoveryStrategy -[$method_call]> Peer2Frontend: register_peer() + return + ref over Peer1Frontend, Peer2Frontend: Establish Connection + Peer2DiscoveryStrategy -[$method_call]> Peer2Frontend: are_all_peers_connected() + return + end +else + loop are_all_peers_connected() + Peer1DiscoveryStrategy x[$udp]- Peer2DiscoveryStrategy: PingMessage + end +end + +@enduml \ No newline at end of file diff --git a/doc/design/udf_discovery_and_communication.rst b/doc/design/udf_discovery_and_communication.rst index 2b35796a..2f8ae0be 100644 --- a/doc/design/udf_discovery_and_communication.rst +++ b/doc/design/udf_discovery_and_communication.rst @@ -1,80 +1,59 @@ UDF Discovery and Communication =============================== -=============== -Local Discovery -=============== +=================== +Establish Connection +=================== -********* -Overview: -********* +* We use a protocol similar to the TCP Handshake for establishing the connection +* However, our protocol has two different requirements compared to TCP: -- UDP Broadcast for the initial `PingMessage` with connection information for the receiving socket of the reliable network -- After receiving `PingMessage`: + * Two peers can establish the connection at the same time + * In case of lost messages, one of the peers in a connection can successful terminate - - Create sending socket for peer to its receiving socket of the reliable network - - Send a `ReadyToReceiveMessage` with our connection information including the receiving socket port - over our sending socket for it to inform the peer that we are ready to receive from it +* To handle, these two requirements, we add the following modification: -- After receiving `ReadyToReceiveMessage` on our receiving socket: + * We allow both peers to send a synchronize at the same time + * When a peer receives the `SynchronizeConnectionMessage` from the second peer - - If not yet discovered, we create a sending socket for the peer to its receiving socket of the reliable network - - Send a `ReadyToReceiveMessage` with our connection information including the receiving socket port - over our sending socket for it, in case we didn't get its `PingMessage`. - - If we didn't get a `ReadyToReceiveMessage` message after a certain time , - yet, we send a `AreYouReadyToReceiveMessage` to a discovered peer + * It sends first a `SynchronizeConnectionMessage` and `AcknowledgeConnectionMessage` back + * It can mark the second peer as ready, after it waited for the peer_is_ready_wait_time - - This should prevent a stuck handshake if we should lose a `ReadyToReceiveMessage` for whatever reason +* Both peers register each other: -- After receiving `AreYouReadyToReceiveMessage`: +.. image:: establish_connection/sequence/both_peers_receive_register_peer.png - - If not yet discovered, we create a sending socket for the peer to its receiving socket of the reliable network - - Send a `ReadyToReceiveMessage` with our connection information including the receiving socket port - over our sending socket for it, in case we didn't get its `PingMessage`. +* One peer registers the other peer: -.. image:: udf_communication_simple_overview.drawio.png +.. image:: establish_connection/sequence/one_peer_receives_register_peer.png -******** -Details: -******** +* Both peers register each other, one peer loses the `SynchronizeConnectionMessage`: -We separated the Local Discovery into two components. The component `LocalDiscoveryStrategy` implements -the discovery via UDP Broadcast. The component `PeerCommunicator` handles the reliable network. +.. image:: establish_connection/sequence/both_peers_receive_register_peer_one_peer_loses_synchonize.png -If the `LocalDiscoveryStrategy` receives a `PingMessage` via UDP it registers the connection info for -the reliable network of the peer with the `PeerCommunicator`. +* Both peers register each other, both lose the `SynchronizeConnectionMessage`: -The `PeerCommunicator` then handles the reliable network communication. -This includes sending and receiving the `ReadyToReceiveMessage` or `AreYouReadyToReceiveMessage`. -It also provides an interface for the user of the library to check if all peers are connected, which peers are there -and to send and receive message from these peers. +.. image:: establish_connection/sequence/both_peers_receive_register_peer_both_peers_lose_synchonize.png -The `PeerCommunicator` can be used with different discovery strategies. -The `LocalDiscoveryStrategy` is one, but the `GlobalDiscoveryStrategy` can use it as well -to form the reliable networks between the leaders. +* State diagram: -The current implementation of the `PeerCommunicator` use `ZMQ` for the reliable communication, -because it abstracts away the low-level network. It provides: +.. image:: establish_connection/state_diagram.png -- A message-based interface, instead the stream-based interface of TCP. -- Asynchronous message queue, instead of synchronous TCP socket +======================== +Local Discovery Strategy +======================== - - Being asynchronous means that the timings of the physical connection setup and tear down, - reconnect and effective delivery are transparent to the user and organized by ZeroMQ itself. - - Further, messages may be queued in the event that a peer is unavailable to receive them. +- The Local Discovery Strategy sends `PingMessage` with connection information + for establishing the connection via UDP Broadcast. +- When a peer receives a `PingMessage` from another peer. + it registers the other peer and treis to establish a connection +- The strategy sends and receives UDP Broadcast messages until all other peers are connected -We further split up the `PeerCommunicator` into a frontend which is called `PeerCommunicator` -and a `BackgroundListener` which runs in a thread. The `BackgroundListener` is also split into the -`BackgroundListenerInterface` and the `BackgroundListenerThread` to simplify the interaction between it -and the `PeerCommunicator` +* Both peers receive `PingMessage`: -The `BackgroundListenerThread` listens for incoming messages from other peers or the frontend and -forwards messages to the frontend. +.. image:: establish_connection/both_peer_receive_ping.png -Here a detailed overview of the information flow: +* One peer receive `PingMessage`: -.. image:: udf_communication_detail_overview.drawio.png +.. image:: establish_connection/one_peer_receives_ping.png -Here the state machines for the BackgroundListener and Frontend - -.. image:: peer_communicator_state_machine.drawio.png diff --git a/exasol_advanced_analytics_framework/udf_communication/ip_address.py b/exasol_advanced_analytics_framework/udf_communication/ip_address.py index cebde836..02ab2ca6 100644 --- a/exasol_advanced_analytics_framework/udf_communication/ip_address.py +++ b/exasol_advanced_analytics_framework/udf_communication/ip_address.py @@ -1,5 +1,3 @@ -import dataclasses - from pydantic import BaseModel diff --git a/exasol_advanced_analytics_framework/udf_communication/local_discovery_socket.py b/exasol_advanced_analytics_framework/udf_communication/local_discovery_socket.py index 63c4489a..a852cdcf 100644 --- a/exasol_advanced_analytics_framework/udf_communication/local_discovery_socket.py +++ b/exasol_advanced_analytics_framework/udf_communication/local_discovery_socket.py @@ -1,6 +1,4 @@ import socket -import time -from typing import Optional from exasol_advanced_analytics_framework.udf_communication.ip_address import IPAddress, Port diff --git a/exasol_advanced_analytics_framework/udf_communication/messages.py b/exasol_advanced_analytics_framework/udf_communication/messages.py index 35c9a598..dd331895 100644 --- a/exasol_advanced_analytics_framework/udf_communication/messages.py +++ b/exasol_advanced_analytics_framework/udf_communication/messages.py @@ -1,4 +1,4 @@ -from typing import Literal, Union, ForwardRef, List, Optional +from typing import Literal, Union from pydantic import BaseModel @@ -35,16 +35,20 @@ class MyConnectionInfoMessage(BaseModel, frozen=True): my_connection_info: ConnectionInfo -class WeAreReadyToReceiveMessage(BaseModel, frozen=True): - message_type: Literal["WeAreReadyToReceiveMessage"] = "WeAreReadyToReceiveMessage" +class SynchronizeConnectionMessage(BaseModel, frozen=True): + message_type: Literal["SynchronizeConnectionMessage"] = "SynchronizeConnectionMessage" source: ConnectionInfo -class AreYouReadyToReceiveMessage(BaseModel, frozen=True): - message_type: Literal["AreYouReadyToReceiveMessage"] = "AreYouReadyToReceiveMessage" +class AcknowledgeConnectionMessage(BaseModel, frozen=True): + message_type: Literal["AcknowledgeConnectionMessage"] = "AcknowledgeConnectionMessage" source: ConnectionInfo +class TimeoutMessage(BaseModel, frozen=True): + message_type: Literal["TimeoutMessage"] = "TimeoutMessage" + + class Message(BaseModel, frozen=True): __root__: Union[ PingMessage, @@ -52,7 +56,8 @@ class Message(BaseModel, frozen=True): StopMessage, PayloadMessage, MyConnectionInfoMessage, - WeAreReadyToReceiveMessage, - AreYouReadyToReceiveMessage, - PeerIsReadyToReceiveMessage + PeerIsReadyToReceiveMessage, + SynchronizeConnectionMessage, + AcknowledgeConnectionMessage, + TimeoutMessage ] diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/abort_timeout_sender.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/abort_timeout_sender.py new file mode 100644 index 00000000..3be010fb --- /dev/null +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/abort_timeout_sender.py @@ -0,0 +1,51 @@ +import structlog +from structlog.typing import FilteringBoundLogger + +from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo +from exasol_advanced_analytics_framework.udf_communication.messages import TimeoutMessage +from exasol_advanced_analytics_framework.udf_communication.peer import Peer +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.timer import Timer +from exasol_advanced_analytics_framework.udf_communication.serialization import serialize_message +from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import Socket + +LOGGER: FilteringBoundLogger = structlog.get_logger() + + +class AbortTimeoutSender: + def __init__(self, + my_connection_info: ConnectionInfo, + peer: Peer, + out_control_socket: Socket, + timer: Timer): + self._timer = timer + self._out_control_socket = out_control_socket + self._finished = False + self._logger = LOGGER.bind( + peer=peer.dict(), + my_connection_info=my_connection_info.dict()) + + def reset_timer(self): + self._logger.info("reset_timer") + self._timer.reset_timer() + + def stop(self): + self._logger.info("stop") + self._finished = True + + def send_if_necessary(self): + self._logger.debug("send_if_necessary") + should_we_send = self._should_we_send() + if should_we_send: + self._finished = True + self._send_timeout_to_frontend() + + def _should_we_send(self): + is_time = self._timer.is_time() + result = is_time and not self._finished + return result + + def _send_timeout_to_frontend(self): + self._logger.debug("send") + message = TimeoutMessage() + serialized_message = serialize_message(message) + self._out_control_socket.send(serialized_message) diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_listener_interface.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_listener_interface.py index 37a09746..ede4c63a 100644 --- a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_listener_interface.py +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_listener_interface.py @@ -1,5 +1,4 @@ import threading -import traceback from typing import Optional, Iterator import structlog @@ -12,6 +11,7 @@ from exasol_advanced_analytics_framework.udf_communication.peer import Peer from exasol_advanced_analytics_framework.udf_communication.peer_communicator.background_listener_thread import \ BackgroundListenerThread +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.clock import Clock from exasol_advanced_analytics_framework.udf_communication.serialization import deserialize_message, serialize_message from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import SocketFactory, \ SocketType, Socket, PollerFlag @@ -25,11 +25,16 @@ def __init__(self, name: str, socket_factory: SocketFactory, listen_ip: IPAddress, - group_identifier: str): + group_identifier: str, + clock: Clock, + poll_timeout_in_ms: int, + synchronize_timeout_in_ms: int, + abort_timeout_in_ms: int, + peer_is_ready_wait_time_in_ms: int, + send_socket_linger_time_in_ms:int, + trace_logging: bool): self._name = name self._logger = LOGGER.bind( - module_name=__name__, - clazz=self.__class__.__name__, name=self._name, group_identifier=group_identifier ) @@ -43,6 +48,13 @@ def __init__(self, group_identifier=group_identifier, out_control_socket_address=out_control_socket_address, in_control_socket_address=in_control_socket_address, + clock=clock, + poll_timeout_in_ms=poll_timeout_in_ms, + synchronize_timeout_in_ms=synchronize_timeout_in_ms, + abort_timeout_in_ms=abort_timeout_in_ms, + peer_is_ready_wait_time_in_ms=peer_is_ready_wait_time_in_ms, + send_socket_linger_time_in_ms=send_socket_linger_time_in_ms, + trace_logging=trace_logging ) self._thread = threading.Thread(target=self._background_listener_run.run) self._thread.daemon = True @@ -70,10 +82,7 @@ def _set_my_connection_info(self): assert isinstance(specific_message_obj, MyConnectionInfoMessage) self._my_connection_info = specific_message_obj.my_connection_info except Exception as e: - self._logger.exception("Exception", - location="_set_my_connection_info", - raw_message=message, - exception=traceback.format_exc()) + self._logger.exception("Exception", raw_message=message) @property def my_connection_info(self) -> ConnectionInfo: @@ -95,13 +104,9 @@ def receive_messages(self, timeout_in_milliseconds: Optional[int] = 0) -> Iterat timeout_in_milliseconds = 0 yield specific_message_obj except Exception as e: - self._logger.exception("Exception", - location="receive_messages", - raw_message=message, - exception=traceback.format_exc()) + self._logger.exception("Exception", raw_message=message) def close(self): - logger = self._logger.bind(location="close") self._logger.info("start") self._send_stop() self._thread.join() @@ -110,5 +115,6 @@ def close(self): self._logger.info("end") def _send_stop(self): + self._logger.info("_send_stop") stop_message = StopMessage() self._in_control_socket.send(serialize_message(stop_message)) diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_listener_thread.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_listener_thread.py index 029ae6ef..aa8ec490 100644 --- a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_listener_thread.py +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_listener_thread.py @@ -1,5 +1,4 @@ import enum -import traceback from typing import Dict, List import structlog @@ -8,10 +7,11 @@ from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo from exasol_advanced_analytics_framework.udf_communication.ip_address import IPAddress, Port from exasol_advanced_analytics_framework.udf_communication.messages import Message, StopMessage, RegisterPeerMessage, \ - WeAreReadyToReceiveMessage, PayloadMessage, MyConnectionInfoMessage, AreYouReadyToReceiveMessage + PayloadMessage, MyConnectionInfoMessage, SynchronizeConnectionMessage, AcknowledgeConnectionMessage from exasol_advanced_analytics_framework.udf_communication.peer import Peer from exasol_advanced_analytics_framework.udf_communication.peer_communicator.background_peer_state import \ BackgroundPeerState +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.clock import Clock from exasol_advanced_analytics_framework.udf_communication.serialization import deserialize_message, serialize_message from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import SocketFactory, \ SocketType, Socket, PollerFlag, Frame @@ -31,20 +31,28 @@ def __init__(self, group_identifier: str, out_control_socket_address: str, in_control_socket_address: str, - poll_timeout_in_seconds: int = 1, - reminder_timeout_in_seconds: float = 1): - self._wait_time_between_reminder_in_seconds = reminder_timeout_in_seconds + clock: Clock, + poll_timeout_in_ms: int, + synchronize_timeout_in_ms: int, + abort_timeout_in_ms: int, + peer_is_ready_wait_time_in_ms: int, + send_socket_linger_time_in_ms: int, + trace_logging: bool): + self._send_socket_linger_time_in_ms = send_socket_linger_time_in_ms + self._trace_logging = trace_logging + self._clock = clock + self._peer_is_ready_wait_time_in_ms = peer_is_ready_wait_time_in_ms + self._abort_timeout_in_ms = abort_timeout_in_ms + self._synchronize_timeout_in_ms = synchronize_timeout_in_ms self._name = name self._logger = LOGGER.bind( - module_name=__name__, - clazz=self.__class__.__name__, name=self._name, group_identifier=group_identifier) self._group_identifier = group_identifier self._listen_ip = listen_ip self._in_control_socket_address = in_control_socket_address self._out_control_socket_address = out_control_socket_address - self._poll_timeout_in_seconds = poll_timeout_in_seconds + self._poll_timeout_in_ms = poll_timeout_in_ms self._socket_factory = socket_factory self._status = BackgroundListenerThread.Status.RUNNING @@ -61,14 +69,13 @@ def run(self): self._close() def _close(self): - logger = self._logger.bind(location="close") - logger.info("start") + self._logger.info("start") self._out_control_socket.close(linger=0) self._in_control_socket.close(linger=0) for peer_state in self._peer_state.values(): peer_state.close() self._listener_socket.close(linger=0) - logger.info("end") + self._logger.info("end") def _create_listener_socket(self): self._listener_socket: Socket = self._socket_factory.create_socket(SocketType.ROUTER) @@ -90,10 +97,9 @@ def _create_poller(self): self.poller.register(self._listener_socket, flags=PollerFlag.POLLIN) def _run_message_loop(self): - log = self._logger.bind(location="_run_message_loop") try: while self._status == BackgroundListenerThread.Status.RUNNING: - poll = self.poller.poll(timeout_in_ms=self._poll_timeout_in_seconds * 1000) + poll = self.poller.poll(timeout_in_ms=self._poll_timeout_in_ms) if self._in_control_socket in poll and PollerFlag.POLLIN in poll[self._in_control_socket]: message = self._in_control_socket.receive() self._status = self._handle_control_message(message) @@ -102,12 +108,12 @@ def _run_message_loop(self): self._handle_listener_message(message) if self._status == BackgroundListenerThread.Status.RUNNING: for peer_state in self._peer_state.values(): - peer_state._send_are_you_ready_to_receive_if_necassary() + peer_state.resend_if_necessary() except Exception as e: - log.exception("Exception", exception=traceback.format_exc()) + self._logger.exception("Exception in message loop") + def _handle_control_message(self, message: bytes) -> Status: - logger = self._logger.bind(location="_handle_control_message") try: message_obj: Message = deserialize_message(message, Message) specific_message_obj = message_obj.__root__ @@ -116,63 +122,62 @@ def _handle_control_message(self, message: bytes) -> Status: elif isinstance(specific_message_obj, RegisterPeerMessage): self._add_peer(specific_message_obj.peer) else: - logger.error( - "Unknown message type", - message=specific_message_obj.dict()) + self._logger.error("Unknown message type", message=specific_message_obj.dict()) except Exception as e: - logger.exception( - "Could not deserialize message", - message=message, - exception=traceback.format_exc() - ) + self._logger.exception("Exception during handling message", message=message) return BackgroundListenerThread.Status.RUNNING def _add_peer(self, peer): + if peer.connection_info.group_identifier != self._my_connection_info.group_identifier: + self._logger.error("Peer belongs to a different group", + my_connection_info=self._my_connection_info.dict(), + peer=peer.dict()) + raise ValueError("Peer belongs to a different group") if peer not in self._peer_state: - self._peer_state[peer] = BackgroundPeerState( + self._peer_state[peer] = BackgroundPeerState.create( my_connection_info=self._my_connection_info, out_control_socket=self._out_control_socket, socket_factory=self._socket_factory, peer=peer, - reminder_timeout_in_seconds=self._wait_time_between_reminder_in_seconds + clock=self._clock, + peer_is_ready_wait_time_in_ms=self._peer_is_ready_wait_time_in_ms, + abort_timeout_in_ms=self._abort_timeout_in_ms, + synchronize_timeout_in_ms=self._synchronize_timeout_in_ms, + send_socket_linger_time_in_ms=self._send_socket_linger_time_in_ms ) def _handle_listener_message(self, message: List[Frame]): logger = self._logger.bind( - location="_handle_listener_message", sender=message[0].to_bytes() ) + message_content_bytes = message[1].to_bytes() try: - message_obj: Message = deserialize_message(message[1].to_bytes(), Message) + message_obj: Message = deserialize_message(message_content_bytes, Message) specific_message_obj = message_obj.__root__ - if isinstance(specific_message_obj, WeAreReadyToReceiveMessage): - self._handle_we_are_ready_to_receive(specific_message_obj) - elif isinstance(specific_message_obj, AreYouReadyToReceiveMessage): - self._handle_are_you_ready_to_receive(specific_message_obj) + if isinstance(specific_message_obj, SynchronizeConnectionMessage): + self._handle_synchronize_connection(specific_message_obj) + elif isinstance(specific_message_obj, AcknowledgeConnectionMessage): + self._handle_acknowledge_connection(specific_message_obj) elif isinstance(specific_message_obj, PayloadMessage): self._handle_payload_message(specific_message_obj, message) else: logger.error("Unknown message type", message=specific_message_obj.dict()) except Exception as e: - logger.exception( - "Could not deserialize message", - message=message[1].to_bytes(), - exception=traceback.format_exc() - ) + logger.exception("Exception during handling message", message_content=message_content_bytes) def _handle_payload_message(self, message: PayloadMessage, frames: List[Frame]): peer = Peer(connection_info=message.source) self._peer_state[peer].forward_payload(frames[2:]) - def _handle_we_are_ready_to_receive(self, message: WeAreReadyToReceiveMessage): + def _handle_synchronize_connection(self, message: SynchronizeConnectionMessage): peer = Peer(connection_info=message.source) self._add_peer(peer) - self._peer_state[peer].received_peer_is_ready_to_receive() + self._peer_state[peer].received_synchronize_connection() - def _handle_are_you_ready_to_receive(self, message: AreYouReadyToReceiveMessage): + def _handle_acknowledge_connection(self, message: AcknowledgeConnectionMessage): peer = Peer(connection_info=message.source) self._add_peer(peer) - self._peer_state[peer].received_are_you_ready_to_receive() + self._peer_state[peer].received_acknowledge_connection() def _set_my_connection_info(self, port: int): self._my_connection_info = ConnectionInfo( diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_peer_state.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_peer_state.py index a9b42bc0..ce797f23 100644 --- a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_peer_state.py +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/background_peer_state.py @@ -1,105 +1,120 @@ -import contextlib -import time -from typing import Optional, Generator, List +from typing import List import structlog from structlog.typing import FilteringBoundLogger from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo -from exasol_advanced_analytics_framework.udf_communication.messages import Message, WeAreReadyToReceiveMessage, \ - AreYouReadyToReceiveMessage, PeerIsReadyToReceiveMessage +from exasol_advanced_analytics_framework.udf_communication.messages import Message, AcknowledgeConnectionMessage from exasol_advanced_analytics_framework.udf_communication.peer import Peer +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.abort_timeout_sender import \ + AbortTimeoutSender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.clock import Clock from exasol_advanced_analytics_framework.udf_communication.peer_communicator.get_peer_receive_socket_name import \ get_peer_receive_socket_name -from exasol_advanced_analytics_framework.udf_communication.serialization import serialize_message -from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import SocketFactory, \ - SocketType, Socket, Frame +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.peer_is_ready_sender import \ + PeerIsReadySender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.sender import Sender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.synchronize_connection_sender import \ + SynchronizeConnectionSender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.timer import Timer +from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import Socket, SocketFactory, \ + SocketType, Frame -LOGGER: FilteringBoundLogger = structlog.get_logger(__name__) +LOGGER: FilteringBoundLogger = structlog.get_logger() class BackgroundPeerState: + @classmethod + def create( + cls, + my_connection_info: ConnectionInfo, + out_control_socket: Socket, + socket_factory: SocketFactory, + peer: Peer, + clock: Clock, + synchronize_timeout_in_ms: int, + abort_timeout_in_ms: int, + peer_is_ready_wait_time_in_ms: int, + send_socket_linger_time_in_ms: int + ): + sender = Sender(my_connection_info=my_connection_info, + socket_factory=socket_factory, + peer=peer, + send_socket_linger_time_in_ms=send_socket_linger_time_in_ms) + synchronize_connection_sender = SynchronizeConnectionSender( + my_connection_info=my_connection_info, + peer=peer, + sender=sender, + timer=Timer(clock=clock, timeout_in_ms=synchronize_timeout_in_ms) + ) + abort_timeout_sender = AbortTimeoutSender( + out_control_socket=out_control_socket, + timer=Timer(clock=clock, timeout_in_ms=abort_timeout_in_ms), + my_connection_info=my_connection_info, + peer=peer + ) + peer_is_ready_sender = PeerIsReadySender( + out_control_socket=out_control_socket, + timer=Timer(clock=clock, timeout_in_ms=peer_is_ready_wait_time_in_ms), + peer=peer, + my_connection_info=my_connection_info, + ) + peer_state = cls( + my_connection_info=my_connection_info, + socket_factory=socket_factory, + peer=peer, + sender=sender, + synchronize_connection_sender=synchronize_connection_sender, + abort_timeout_sender=abort_timeout_sender, + peer_is_ready_sender=peer_is_ready_sender + ) + return peer_state + def __init__(self, my_connection_info: ConnectionInfo, - out_control_socket: Socket, socket_factory: SocketFactory, peer: Peer, - reminder_timeout_in_seconds: float = 1): - self._out_control_socket = out_control_socket + sender: Sender, + synchronize_connection_sender: SynchronizeConnectionSender, + abort_timeout_sender: AbortTimeoutSender, + peer_is_ready_sender: PeerIsReadySender): self._my_connection_info = my_connection_info - self._wait_time_between_reminder_in_seconds = reminder_timeout_in_seconds self._peer = peer self._socket_factory = socket_factory - self._peer_can_receive_from_us = False - self._last_send_ready_to_receive_timestamp_in_seconds: Optional[float] = None - self._logger = LOGGER.bind( - module_name=__name__, - clazz=self.__class__.__name__, - peer=self._peer, - my_connection_info=self._my_connection_info, - ) self._create_receive_socket() - self._send_we_are_ready_to_receive() + self._sender = sender + self._synchronize_connection_sender = synchronize_connection_sender + self._abort_timeout_sender = abort_timeout_sender + self._peer_is_ready_sender = peer_is_ready_sender + self._synchronize_connection_sender.send_if_necessary(force=True) + self._logger = LOGGER.bind( + peer=self._peer.dict(), + my_connection_info=self._my_connection_info.dict()) def _create_receive_socket(self): self._receive_socket = self._socket_factory.create_socket(SocketType.PAIR) receive_socket_address = get_peer_receive_socket_name(self._peer) self._receive_socket.bind(receive_socket_address) - @contextlib.contextmanager - def _create_send_socket(self) -> Generator[Socket, None, None]: - send_socket: Socket - with self._socket_factory.create_socket(SocketType.DEALER) as send_socket: - send_socket.connect( - f"tcp://{self._peer.connection_info.ipaddress.ip_address}:{self._peer.connection_info.port.port}") - yield send_socket - - def _is_time_to_send_are_you_ready_to_receive(self): - current_timestamp_in_seconds = time.monotonic() - if self._last_send_ready_to_receive_timestamp_in_seconds is not None: - diff = current_timestamp_in_seconds - self._last_send_ready_to_receive_timestamp_in_seconds - if diff > self._wait_time_between_reminder_in_seconds: - self._last_send_ready_to_receive_timestamp_in_seconds = current_timestamp_in_seconds - return True - else: - self._last_send_ready_to_receive_timestamp_in_seconds = current_timestamp_in_seconds - return False - - def _send_are_you_ready_to_receive_if_necassary(self): - if not self._peer_can_receive_from_us: - if self._is_time_to_send_are_you_ready_to_receive(): - self._logger.info("Send AreYouReadyToReceiveMessage", peer=self._peer, - my_connection_info=self._my_connection_info) - message = Message(__root__=AreYouReadyToReceiveMessage(source=self._my_connection_info)) - self._send(message) - - def _send_we_are_ready_to_receive(self): - message = Message(__root__=WeAreReadyToReceiveMessage(source=self._my_connection_info)) - self._send(message) - - def received_peer_is_ready_to_receive(self): - self._handle_peer_is_ready_to_receive() - - def received_are_you_ready_to_receive(self): - self._handle_peer_is_ready_to_receive() - self._send_we_are_ready_to_receive() - - def _handle_peer_is_ready_to_receive(self): - if not self._peer_can_receive_from_us: - self._send_peer_is_ready_to_frontend() - self._peer_can_receive_from_us = True - - def _send(self, message: Message): - send_socket: Socket - with self._create_send_socket() as send_socket: - serialized_message = serialize_message(message.__root__) - send_socket.send(serialized_message) - - def _send_peer_is_ready_to_frontend(self): - message = PeerIsReadyToReceiveMessage(peer=self._peer) - serialized_message = serialize_message(message) - self._out_control_socket.send(serialized_message) + def resend_if_necessary(self): + self._logger.debug("resend_if_necessary") + self._synchronize_connection_sender.send_if_necessary() + self._abort_timeout_sender.send_if_necessary() + self._peer_is_ready_sender.send_if_necessary() + + def received_synchronize_connection(self): + self._logger.debug("received_synchronize_connection") + self._peer_is_ready_sender.enable() + self._peer_is_ready_sender.reset_timer() + self._abort_timeout_sender.stop() + self._sender.send(Message(__root__=AcknowledgeConnectionMessage(source=self._my_connection_info))) + + def received_acknowledge_connection(self): + self._logger.debug("received_acknowledge_connection") + self._abort_timeout_sender.stop() + self._synchronize_connection_sender.stop() + self._peer_is_ready_sender.send_if_necessary(force=True) def forward_payload(self, frames: List[Frame]): self._receive_socket.send_multipart(frames) diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/clock.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/clock.py new file mode 100644 index 00000000..6f7b6962 --- /dev/null +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/clock.py @@ -0,0 +1,7 @@ +import time + + +class Clock(): + def current_timestamp_in_ms(self) -> int: + timestamp = time.monotonic_ns() // 10 ** 6 + return timestamp diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/get_peer_receive_socket_name.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/get_peer_receive_socket_name.py index f6a2c69b..dc6e5cf4 100644 --- a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/get_peer_receive_socket_name.py +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/get_peer_receive_socket_name.py @@ -5,6 +5,6 @@ def get_peer_receive_socket_name(peer: Peer) -> str: quoted_ip_address = urllib.parse.quote_plus(peer.connection_info.ipaddress.ip_address) - quoted_port = urllib.parse.quote_plus(str(peer.connection_info.port)) + quoted_port = urllib.parse.quote_plus(str(peer.connection_info.port.port)) quoted_group_identifier = peer.connection_info.group_identifier return f"inproc://peer/{quoted_group_identifier}/{quoted_ip_address}/{quoted_port}" diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/peer_communicator.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/peer_communicator.py index 7adad92b..66398e5a 100644 --- a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/peer_communicator.py +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/peer_communicator.py @@ -6,10 +6,11 @@ from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo from exasol_advanced_analytics_framework.udf_communication.ip_address import IPAddress -from exasol_advanced_analytics_framework.udf_communication.messages import PeerIsReadyToReceiveMessage +from exasol_advanced_analytics_framework.udf_communication.messages import PeerIsReadyToReceiveMessage, TimeoutMessage from exasol_advanced_analytics_framework.udf_communication.peer import Peer from exasol_advanced_analytics_framework.udf_communication.peer_communicator.background_listener_interface import \ BackgroundListenerInterface +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.clock import Clock from exasol_advanced_analytics_framework.udf_communication.peer_communicator.frontend_peer_state import \ FrontendPeerState from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import SocketFactory, \ @@ -36,35 +37,50 @@ def __init__(self, number_of_peers: int, listen_ip: IPAddress, group_identifier: str, - socket_factory: SocketFactory): + socket_factory: SocketFactory, + poll_timeout_in_ms: int = 500, + synchronize_timeout_in_ms: int = 1000, + abort_timeout_in_ms: int = 120000, + peer_is_ready_wait_time_in_ms: int = 2000, + send_socket_linger_time_in_ms: int = 100, + clock: Clock = Clock(), + trace_logging: bool = False): self._socket_factory = socket_factory self._name = name - self._log_info = dict(module_name=__name__, - clazz=self.__class__.__name__, - name=self._name, - group_identifier=group_identifier) - self._logger = LOGGER.bind(**self._log_info) + self._logger = LOGGER.bind( + name=self._name, + group_identifier=group_identifier + ) self._number_of_peers = number_of_peers self._background_listener = BackgroundListenerInterface( name=self._name, socket_factory=self._socket_factory, listen_ip=listen_ip, - group_identifier=group_identifier) + group_identifier=group_identifier, + clock=clock, + poll_timeout_in_ms=poll_timeout_in_ms, + synchronize_timeout_in_ms=synchronize_timeout_in_ms, + abort_timeout_in_ms=abort_timeout_in_ms, + peer_is_ready_wait_time_in_ms=peer_is_ready_wait_time_in_ms, + send_socket_linger_time_in_ms=send_socket_linger_time_in_ms, + trace_logging=trace_logging + ) self._my_connection_info = self._background_listener.my_connection_info self._peer_states: Dict[Peer, FrontendPeerState] = {} def _handle_messages(self, timeout_in_milliseconds: Optional[int] = 0): - if not self._are_all_peers_connected(): - for message in self._background_listener.receive_messages(timeout_in_milliseconds): - if isinstance(message, PeerIsReadyToReceiveMessage): - peer = message.peer - self._add_peer_state(peer) - self._peer_states[peer].received_peer_is_ready_to_receive() - else: - self._logger.error( - "Unknown message", - location="_handle_messages", - message=message.dict()) + if self._are_all_peers_connected(): + return + + for message in self._background_listener.receive_messages(timeout_in_milliseconds): + if isinstance(message, PeerIsReadyToReceiveMessage): + peer = message.peer + self._add_peer_state(peer) + self._peer_states[peer].received_peer_is_ready_to_receive() + elif isinstance(message, TimeoutMessage): + raise TimeoutError() + else: + self._logger.error("Unknown message", message=message.dict()) def _add_peer_state(self, peer): if peer not in self._peer_states: diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/peer_is_ready_sender.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/peer_is_ready_sender.py new file mode 100644 index 00000000..0d7eda6d --- /dev/null +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/peer_is_ready_sender.py @@ -0,0 +1,63 @@ +import structlog +from structlog.typing import FilteringBoundLogger + +from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo +from exasol_advanced_analytics_framework.udf_communication.messages import PeerIsReadyToReceiveMessage +from exasol_advanced_analytics_framework.udf_communication.peer import Peer +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.timer import Timer +from exasol_advanced_analytics_framework.udf_communication.serialization import serialize_message +from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import Socket + +LOGGER: FilteringBoundLogger = structlog.get_logger() + + +import enum + + +class _State(enum.IntFlag): + Init = enum.auto() + Enabled = enum.auto() + Finished = enum.auto() + + +class PeerIsReadySender: + + + def __init__(self, + out_control_socket: Socket, + peer: Peer, + my_connection_info: ConnectionInfo, + timer: Timer): + self._timer = timer + self._peer = peer + self._out_control_socket = out_control_socket + self._state = _State.Init + self._logger = LOGGER.bind( + peer=self._peer.dict(), + my_connection_info=my_connection_info.dict()) + + def enable(self): + self._logger.debug("enable") + self._state |= _State.Enabled + + def reset_timer(self): + self._logger.debug("reset_timer") + self._timer.reset_timer() + + def send_if_necessary(self, force=False): + self._logger.debug("send_if_necessary") + should_we_send = self._should_we_send() + if should_we_send or force: + self._state |= _State.Finished + self._send_peer_is_ready_to_frontend() + + def _should_we_send(self): + is_time = self._timer.is_time() + result = is_time and (_State.Finished not in self._state) and (_State.Enabled in self._state) + return result + + def _send_peer_is_ready_to_frontend(self): + self._logger.debug("send") + message = PeerIsReadyToReceiveMessage(peer=self._peer) + serialized_message = serialize_message(message) + self._out_control_socket.send(serialized_message) diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/sender.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/sender.py new file mode 100644 index 00000000..025652f7 --- /dev/null +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/sender.py @@ -0,0 +1,49 @@ +from typing import Optional + +import structlog +from structlog.typing import FilteringBoundLogger + +from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo +from exasol_advanced_analytics_framework.udf_communication.messages import Message +from exasol_advanced_analytics_framework.udf_communication.peer import Peer +from exasol_advanced_analytics_framework.udf_communication.serialization import serialize_message +from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import SocketFactory, \ + Socket, SocketType + +LOGGER: FilteringBoundLogger = structlog.get_logger(__name__) + + +class Sender: + def __init__(self, + my_connection_info: ConnectionInfo, + socket_factory: SocketFactory, + peer: Peer, + send_socket_linger_time_in_ms: int): + self._send_socket_linger_time_in_ms = send_socket_linger_time_in_ms + self._my_connection_info = my_connection_info + self._peer = peer + self._socket_factory = socket_factory + + self._logger = LOGGER.bind( + peer=self._peer, + my_connection_info=self._my_connection_info, + ) + + def create_send_socket(self) -> Socket: + send_socket: Optional[Socket] = None + try: + send_socket = self._socket_factory.create_socket(SocketType.DEALER) + send_socket.connect( + "tcp://{ip}:{port}".format( + ip=self._peer.connection_info.ipaddress.ip_address, + port=self._peer.connection_info.port.port + )) + return send_socket + except Exception: + send_socket.close() + + def send(self, message: Message): + with self.create_send_socket() as send_socket: + serialized_message = serialize_message(message.__root__) + send_socket.send(serialized_message) + send_socket.close(self._send_socket_linger_time_in_ms) diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/synchronize_connection_sender.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/synchronize_connection_sender.py new file mode 100644 index 00000000..ba57f9fa --- /dev/null +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/synchronize_connection_sender.py @@ -0,0 +1,46 @@ +import structlog +from structlog.typing import FilteringBoundLogger + +from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo +from exasol_advanced_analytics_framework.udf_communication.messages import Message, SynchronizeConnectionMessage +from exasol_advanced_analytics_framework.udf_communication.peer import Peer +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.sender import Sender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.timer import Timer + +LOGGER: FilteringBoundLogger = structlog.get_logger() + + +class SynchronizeConnectionSender(): + def __init__(self, + my_connection_info: ConnectionInfo, + peer: Peer, + sender: Sender, + timer: Timer): + self._my_connection_info = my_connection_info + self._timer = timer + self._sender = sender + self._finished = False + self._logger = LOGGER.bind( + peer=peer.dict(), + my_connection_info=my_connection_info.dict()) + + def stop(self): + self._logger.debug("stop") + self._finished = True + + def send_if_necessary(self, force=False): + self._logger.debug("send_if_necessary") + should_we_send = self._should_we_send() + if should_we_send or force: + self._send() + self._timer.reset_timer() + + def _send(self): + self._logger.debug("send") + message = Message(__root__=SynchronizeConnectionMessage(source=self._my_connection_info)) + self._sender.send(message) + + def _should_we_send(self): + is_time = self._timer.is_time() + result = is_time and not self._finished + return result diff --git a/exasol_advanced_analytics_framework/udf_communication/peer_communicator/timer.py b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/timer.py new file mode 100644 index 00000000..e99e56ce --- /dev/null +++ b/exasol_advanced_analytics_framework/udf_communication/peer_communicator/timer.py @@ -0,0 +1,19 @@ +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.clock import Clock + + +class Timer: + + def __init__(self, + clock: Clock, + timeout_in_ms: int): + self._timeout_in_ms = timeout_in_ms + self._clock = clock + self._last_send_timestamp_in_ms = clock.current_timestamp_in_ms() + + def reset_timer(self): + self._last_send_timestamp_in_ms = self._clock.current_timestamp_in_ms() + + def is_time(self): + current_timestamp_in_ms = self._clock.current_timestamp_in_ms() + diff = current_timestamp_in_ms - self._last_send_timestamp_in_ms + return diff > self._timeout_in_ms diff --git a/exasol_advanced_analytics_framework/udf_communication/socket_factory/fault_injection.py b/exasol_advanced_analytics_framework/udf_communication/socket_factory/fault_injection.py index ea0a616b..1fcc3a37 100644 --- a/exasol_advanced_analytics_framework/udf_communication/socket_factory/fault_injection.py +++ b/exasol_advanced_analytics_framework/udf_communication/socket_factory/fault_injection.py @@ -32,8 +32,6 @@ def __init__(self, internal_socket: abstract.Socket, send_fault_probability: flo raise ValueError( f"send_fault_probability needs to be between 0 and 1 (exclusive) was {send_fault_probability}.") self._logger = LOGGER.bind( - module_name=__name__, - clazz=self.__class__.__name__, socket=str(self) ) self._send_fault_probability = send_fault_probability diff --git a/poetry.lock b/poetry.lock index d2f15a18..5f0f2c4b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,40 @@ # This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +[[package]] +name = "bcrypt" +version = "4.0.1" +description = "Modern password hashing for your software and your servers" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "bcrypt-4.0.1-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:b1023030aec778185a6c16cf70f359cbb6e0c289fd564a7cfa29e727a1c38f8f"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:08d2947c490093a11416df18043c27abe3921558d2c03e2076ccb28a116cb6d0"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0eaa47d4661c326bfc9d08d16debbc4edf78778e6aaba29c1bc7ce67214d4410"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae88eca3024bb34bb3430f964beab71226e761f51b912de5133470b649d82344"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:a522427293d77e1c29e303fc282e2d71864579527a04ddcfda6d4f8396c6c36a"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:fbdaec13c5105f0c4e5c52614d04f0bca5f5af007910daa8b6b12095edaa67b3"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ca3204d00d3cb2dfed07f2d74a25f12fc12f73e606fcaa6975d1f7ae69cacbb2"}, + {file = "bcrypt-4.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:089098effa1bc35dc055366740a067a2fc76987e8ec75349eb9484061c54f535"}, + {file = "bcrypt-4.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:e9a51bbfe7e9802b5f3508687758b564069ba937748ad7b9e890086290d2f79e"}, + {file = "bcrypt-4.0.1-cp36-abi3-win32.whl", hash = "sha256:2caffdae059e06ac23fce178d31b4a702f2a3264c20bfb5ff541b338194d8fab"}, + {file = "bcrypt-4.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:8a68f4341daf7522fe8d73874de8906f3a339048ba406be6ddc1b3ccb16fc0d9"}, + {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf4fa8b2ca74381bb5442c089350f09a3f17797829d958fad058d6e44d9eb83c"}, + {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:67a97e1c405b24f19d08890e7ae0c4f7ce1e56a712a016746c8b2d7732d65d4b"}, + {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b3b85202d95dd568efcb35b53936c5e3b3600c7cdcc6115ba461df3a8e89f38d"}, + {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbb03eec97496166b704ed663a53680ab57c5084b2fc98ef23291987b525cb7d"}, + {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:5ad4d32a28b80c5fa6671ccfb43676e8c1cc232887759d1cd7b6f56ea4355215"}, + {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b57adba8a1444faf784394de3436233728a1ecaeb6e07e8c22c8848f179b893c"}, + {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:705b2cea8a9ed3d55b4491887ceadb0106acf7c6387699fca771af56b1cdeeda"}, + {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:2b3ac11cf45161628f1f3733263e63194f22664bf4d0c0f3ab34099c02134665"}, + {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3100851841186c25f127731b9fa11909ab7b1df6fc4b9f8353f4f1fd952fbf71"}, + {file = "bcrypt-4.0.1.tar.gz", hash = "sha256:27d375903ac8261cfe4047f6709d16f7d18d39b1ec92aaf72af989552a650ebd"}, +] + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + [[package]] name = "certifi" version = "2023.5.7" @@ -390,6 +425,18 @@ files = [ {file = "cycler-0.11.0.tar.gz", hash = "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"}, ] +[[package]] +name = "decorator" +version = "5.1.1" +description = "Decorators for Humans" +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, + {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, +] + [[package]] name = "dill" version = "0.3.6" @@ -405,6 +452,39 @@ files = [ [package.extras] graph = ["objgraph (>=1.7.2)"] +[[package]] +name = "docker" +version = "6.1.3" +description = "A Python library for the Docker Engine API." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"}, + {file = "docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20"}, +] + +[package.dependencies] +packaging = ">=14.0" +requests = ">=2.26.0" +urllib3 = ">=1.26.0" +websocket-client = ">=0.32.0" + +[package.extras] +ssh = ["paramiko (>=2.4.3)"] + +[[package]] +name = "docutils" +version = "0.20.1" +description = "Docutils -- Python Documentation Utilities" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6"}, + {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"}, +] + [[package]] name = "exasol-bucketfs" version = "0.6.0" @@ -450,6 +530,39 @@ url = "https://github.com/exasol/data-science-utils-python.git" reference = "main" resolved_reference = "24bf0f86eab327c69d371714c9d5edf73adadf2d" +[[package]] +name = "exasol-integration-test-docker-environment" +version = "1.7.1" +description = "Integration Test Docker Environment for Exasol" +category = "dev" +optional = false +python-versions = ">=3.8,<4" +files = [ + {file = "exasol_integration_test_docker_environment-1.7.1-py3-none-any.whl", hash = "sha256:531bf53a5c60c422850472710d3ad11983d8636cb5edfbb705c86f0a4a69a125"}, + {file = "exasol_integration_test_docker_environment-1.7.1.tar.gz", hash = "sha256:7fb6b2e225673c124e1ebecd1bbc03d313bf56d3915d526496f21ba0ca9fa816"}, +] + +[package.dependencies] +click = ">=7.0" +docker = {version = ">=4.0.0", markers = "sys_platform != \"win32\""} +exasol-bucketfs = ">=0.6.0,<2.0.0" +fabric = ">=3.0.1,<4.0.0" +gitpython = ">=2.1.0" +humanfriendly = ">=4.18" +importlib_resources = ">=5.4.0" +jinja2 = ">=2.10.1" +jsonpickle = ">=1.1" +luigi = ">=2.8.4" +netaddr = ">=0.7.19" +networkx = ">=2.3" +portalocker = ">=2.7.0,<3.0.0" +pydot = ">=1.4.0" +pyexasol = ">=0.25.2,<0.26.0" +pytest = ">=7.2.2,<8.0.0" +requests = ">=2.21.0" +simplejson = ">=3.16.0" +"stopwatch.py" = ">=1.0.0" + [[package]] name = "exasol-udf-mock-python" version = "0.1.0" @@ -473,19 +586,39 @@ resolved_reference = "2088d62a7457fd78d8152a2181d459d9d6bd4b0d" [[package]] name = "exceptiongroup" -version = "1.1.1" +version = "1.1.2" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, - {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, + {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, + {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, ] [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "fabric" +version = "3.1.0" +description = "High level SSH command execution" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "fabric-3.1.0-py3-none-any.whl", hash = "sha256:0a13217db1aa203167376119b0e165081c5906c31e2b2104410685d1310ef8fb"}, + {file = "fabric-3.1.0.tar.gz", hash = "sha256:ea1c5ea3956d196b5990ba720cc8ee457fa1b9c6f265ab3b643ff63b05e8970a"}, +] + +[package.dependencies] +decorator = ">=5" +invoke = ">=2.0" +paramiko = ">=2.4" + +[package.extras] +pytest = ["pytest (>=7)"] + [[package]] name = "fonttools" version = "4.40.0" @@ -544,6 +677,51 @@ ufo = ["fs (>=2.2.0,<3)"] unicode = ["unicodedata2 (>=15.0.0)"] woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] +[[package]] +name = "gitdb" +version = "4.0.10" +description = "Git Object Database" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, + {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.31" +description = "GitPython is a Python library used to interact with Git repositories" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "GitPython-3.1.31-py3-none-any.whl", hash = "sha256:f04893614f6aa713a60cbbe1e6a97403ef633103cdd0ef5eb6efe0deb98dbe8d"}, + {file = "GitPython-3.1.31.tar.gz", hash = "sha256:8ce3bcf69adfdf7c7d503e78fd3b1c492af782d58893b650adb2ac8912ddd573"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[[package]] +name = "humanfriendly" +version = "10.0" +description = "Human friendly output for text interfaces using Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, + {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, +] + +[package.dependencies] +pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_version >= \"3.8\""} + [[package]] name = "idna" version = "3.4" @@ -587,6 +765,18 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "invoke" +version = "2.1.3" +description = "Pythonic task execution" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "invoke-2.1.3-py3-none-any.whl", hash = "sha256:51e86a08d964160e01c44eccd22f50b25842bd96a9c63c11177032594cb86740"}, + {file = "invoke-2.1.3.tar.gz", hash = "sha256:a3b15d52d50bbabd851b8a39582c772180b614000fa1612b4d92484d54d38c6b"}, +] + [[package]] name = "jinja2" version = "3.1.2" @@ -607,14 +797,14 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "joblib" -version = "1.2.0" +version = "1.3.1" description = "Lightweight pipelining with Python functions" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "joblib-1.2.0-py3-none-any.whl", hash = "sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385"}, - {file = "joblib-1.2.0.tar.gz", hash = "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018"}, + {file = "joblib-1.3.1-py3-none-any.whl", hash = "sha256:89cf0529520e01b3de7ac7b74a8102c90d16d54c64b5dd98cafcd14307fdf915"}, + {file = "joblib-1.3.1.tar.gz", hash = "sha256:1f937906df65329ba98013dc9692fe22a4c5e4a648112de500508b18a21b41e3"}, ] [[package]] @@ -712,6 +902,40 @@ files = [ {file = "kiwisolver-1.4.4.tar.gz", hash = "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955"}, ] +[[package]] +name = "lockfile" +version = "0.12.2" +description = "Platform-independent file locking module" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "lockfile-0.12.2-py2.py3-none-any.whl", hash = "sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa"}, + {file = "lockfile-0.12.2.tar.gz", hash = "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799"}, +] + +[[package]] +name = "luigi" +version = "3.3.0" +description = "Workflow mgmgt + task scheduling + dependency resolution." +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "luigi-3.3.0.tar.gz", hash = "sha256:cc8642deb6e22f0601ee8df8ad3de639a81521b7e663a7a5044919185ad2b357"}, +] + +[package.dependencies] +python-daemon = "*" +python-dateutil = ">=2.7.5,<3" +tenacity = ">=8,<9" +tornado = ">=5.0,<7" + +[package.extras] +jsonschema = ["jsonschema"] +prometheus = ["prometheus-client (>=0.5,<0.15)"] +toml = ["toml (<2.0.0)"] + [[package]] name = "markupsafe" version = "2.1.3" @@ -860,42 +1084,73 @@ setuptools = "*" docs = ["mkdocs"] testing = ["pytest"] +[[package]] +name = "netaddr" +version = "0.8.0" +description = "A network address manipulation library for Python" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "netaddr-0.8.0-py2.py3-none-any.whl", hash = "sha256:9666d0232c32d2656e5e5f8d735f58fd6c7457ce52fc21c98d45f2af78f990ac"}, + {file = "netaddr-0.8.0.tar.gz", hash = "sha256:d6cc57c7a07b1d9d2e917aa8b36ae8ce61c35ba3fcd1b83ca31c5a0ee2b5a243"}, +] + +[[package]] +name = "networkx" +version = "3.1" +description = "Python package for creating and manipulating graphs and networks" +category = "dev" +optional = false +python-versions = ">=3.8" +files = [ + {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, + {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, +] + +[package.extras] +default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] +developer = ["mypy (>=1.1)", "pre-commit (>=3.2)"] +doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.13)", "sphinx (>=6.1)", "sphinx-gallery (>=0.12)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"] +test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] + [[package]] name = "numpy" -version = "1.24.3" +version = "1.24.4" description = "Fundamental package for array computing in Python" category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, - {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, - {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, - {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, - {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, - {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, - {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, - {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, - {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, - {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, - {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, + {file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"}, + {file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6"}, + {file = "numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc"}, + {file = "numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5"}, + {file = "numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d"}, + {file = "numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc"}, + {file = "numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2"}, + {file = "numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d"}, + {file = "numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835"}, + {file = "numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2"}, + {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, ] [[package]] @@ -959,6 +1214,28 @@ pytz = ">=2020.1" [package.extras] test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] +[[package]] +name = "paramiko" +version = "3.2.0" +description = "SSH2 protocol library" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "paramiko-3.2.0-py3-none-any.whl", hash = "sha256:df0f9dd8903bc50f2e10580af687f3015bf592a377cd438d2ec9546467a14eb8"}, + {file = "paramiko-3.2.0.tar.gz", hash = "sha256:93cdce625a8a1dc12204439d45033f3261bdb2c201648cfcdc06f9fd0f94ec29"}, +] + +[package.dependencies] +bcrypt = ">=3.2" +cryptography = ">=3.3" +pynacl = ">=1.5" + +[package.extras] +all = ["gssapi (>=1.4.1)", "invoke (>=2.0)", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1.8)"] +gssapi = ["gssapi (>=1.4.1)", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1.8)"] +invoke = ["invoke (>=2.0)"] + [[package]] name = "pastel" version = "0.2.1" @@ -973,78 +1250,66 @@ files = [ [[package]] name = "pillow" -version = "9.5.0" +version = "10.0.0" description = "Python Imaging Library (Fork)" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "Pillow-9.5.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:ace6ca218308447b9077c14ea4ef381ba0b67ee78d64046b3f19cf4e1139ad16"}, - {file = "Pillow-9.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d3d403753c9d5adc04d4694d35cf0391f0f3d57c8e0030aac09d7678fa8030aa"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ba1b81ee69573fe7124881762bb4cd2e4b6ed9dd28c9c60a632902fe8db8b38"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe7e1c262d3392afcf5071df9afa574544f28eac825284596ac6db56e6d11062"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f36397bf3f7d7c6a3abdea815ecf6fd14e7fcd4418ab24bae01008d8d8ca15e"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:252a03f1bdddce077eff2354c3861bf437c892fb1832f75ce813ee94347aa9b5"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:85ec677246533e27770b0de5cf0f9d6e4ec0c212a1f89dfc941b64b21226009d"}, - {file = "Pillow-9.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b416f03d37d27290cb93597335a2f85ed446731200705b22bb927405320de903"}, - {file = "Pillow-9.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1781a624c229cb35a2ac31cc4a77e28cafc8900733a864870c49bfeedacd106a"}, - {file = "Pillow-9.5.0-cp310-cp310-win32.whl", hash = "sha256:8507eda3cd0608a1f94f58c64817e83ec12fa93a9436938b191b80d9e4c0fc44"}, - {file = "Pillow-9.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:d3c6b54e304c60c4181da1c9dadf83e4a54fd266a99c70ba646a9baa626819eb"}, - {file = "Pillow-9.5.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:7ec6f6ce99dab90b52da21cf0dc519e21095e332ff3b399a357c187b1a5eee32"}, - {file = "Pillow-9.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:560737e70cb9c6255d6dcba3de6578a9e2ec4b573659943a5e7e4af13f298f5c"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96e88745a55b88a7c64fa49bceff363a1a27d9a64e04019c2281049444a571e3"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d9c206c29b46cfd343ea7cdfe1232443072bbb270d6a46f59c259460db76779a"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cfcc2c53c06f2ccb8976fb5c71d448bdd0a07d26d8e07e321c103416444c7ad1"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:a0f9bb6c80e6efcde93ffc51256d5cfb2155ff8f78292f074f60f9e70b942d99"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:8d935f924bbab8f0a9a28404422da8af4904e36d5c33fc6f677e4c4485515625"}, - {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fed1e1cf6a42577953abbe8e6cf2fe2f566daebde7c34724ec8803c4c0cda579"}, - {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c1170d6b195555644f0616fd6ed929dfcf6333b8675fcca044ae5ab110ded296"}, - {file = "Pillow-9.5.0-cp311-cp311-win32.whl", hash = "sha256:54f7102ad31a3de5666827526e248c3530b3a33539dbda27c6843d19d72644ec"}, - {file = "Pillow-9.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:cfa4561277f677ecf651e2b22dc43e8f5368b74a25a8f7d1d4a3a243e573f2d4"}, - {file = "Pillow-9.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:965e4a05ef364e7b973dd17fc765f42233415974d773e82144c9bbaaaea5d089"}, - {file = "Pillow-9.5.0-cp312-cp312-win32.whl", hash = "sha256:22baf0c3cf0c7f26e82d6e1adf118027afb325e703922c8dfc1d5d0156bb2eeb"}, - {file = "Pillow-9.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:432b975c009cf649420615388561c0ce7cc31ce9b2e374db659ee4f7d57a1f8b"}, - {file = "Pillow-9.5.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:5d4ebf8e1db4441a55c509c4baa7a0587a0210f7cd25fcfe74dbbce7a4bd1906"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:375f6e5ee9620a271acb6820b3d1e94ffa8e741c0601db4c0c4d3cb0a9c224bf"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99eb6cafb6ba90e436684e08dad8be1637efb71c4f2180ee6b8f940739406e78"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dfaaf10b6172697b9bceb9a3bd7b951819d1ca339a5ef294d1f1ac6d7f63270"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:763782b2e03e45e2c77d7779875f4432e25121ef002a41829d8868700d119392"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:35f6e77122a0c0762268216315bf239cf52b88865bba522999dc38f1c52b9b47"}, - {file = "Pillow-9.5.0-cp37-cp37m-win32.whl", hash = "sha256:aca1c196f407ec7cf04dcbb15d19a43c507a81f7ffc45b690899d6a76ac9fda7"}, - {file = "Pillow-9.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322724c0032af6692456cd6ed554bb85f8149214d97398bb80613b04e33769f6"}, - {file = "Pillow-9.5.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:a0aa9417994d91301056f3d0038af1199eb7adc86e646a36b9e050b06f526597"}, - {file = "Pillow-9.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f8286396b351785801a976b1e85ea88e937712ee2c3ac653710a4a57a8da5d9c"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c830a02caeb789633863b466b9de10c015bded434deb3ec87c768e53752ad22a"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fbd359831c1657d69bb81f0db962905ee05e5e9451913b18b831febfe0519082"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8fc330c3370a81bbf3f88557097d1ea26cd8b019d6433aa59f71195f5ddebbf"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:7002d0797a3e4193c7cdee3198d7c14f92c0836d6b4a3f3046a64bd1ce8df2bf"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:229e2c79c00e85989a34b5981a2b67aa079fd08c903f0aaead522a1d68d79e51"}, - {file = "Pillow-9.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9adf58f5d64e474bed00d69bcd86ec4bcaa4123bfa70a65ce72e424bfb88ed96"}, - {file = "Pillow-9.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:662da1f3f89a302cc22faa9f14a262c2e3951f9dbc9617609a47521c69dd9f8f"}, - {file = "Pillow-9.5.0-cp38-cp38-win32.whl", hash = "sha256:6608ff3bf781eee0cd14d0901a2b9cc3d3834516532e3bd673a0a204dc8615fc"}, - {file = "Pillow-9.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:e49eb4e95ff6fd7c0c402508894b1ef0e01b99a44320ba7d8ecbabefddcc5569"}, - {file = "Pillow-9.5.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:482877592e927fd263028c105b36272398e3e1be3269efda09f6ba21fd83ec66"}, - {file = "Pillow-9.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3ded42b9ad70e5f1754fb7c2e2d6465a9c842e41d178f262e08b8c85ed8a1d8e"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c446d2245ba29820d405315083d55299a796695d747efceb5717a8b450324115"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aca1152d93dcc27dc55395604dcfc55bed5f25ef4c98716a928bacba90d33a3"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:608488bdcbdb4ba7837461442b90ea6f3079397ddc968c31265c1e056964f1ef"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:60037a8db8750e474af7ffc9faa9b5859e6c6d0a50e55c45576bf28be7419705"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:07999f5834bdc404c442146942a2ecadd1cb6292f5229f4ed3b31e0a108746b1"}, - {file = "Pillow-9.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a127ae76092974abfbfa38ca2d12cbeddcdeac0fb71f9627cc1135bedaf9d51a"}, - {file = "Pillow-9.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:489f8389261e5ed43ac8ff7b453162af39c3e8abd730af8363587ba64bb2e865"}, - {file = "Pillow-9.5.0-cp39-cp39-win32.whl", hash = "sha256:9b1af95c3a967bf1da94f253e56b6286b50af23392a886720f563c547e48e964"}, - {file = "Pillow-9.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:77165c4a5e7d5a284f10a6efaa39a0ae8ba839da344f20b111d62cc932fa4e5d"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:833b86a98e0ede388fa29363159c9b1a294b0905b5128baf01db683672f230f5"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aaf305d6d40bd9632198c766fb64f0c1a83ca5b667f16c1e79e1661ab5060140"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0852ddb76d85f127c135b6dd1f0bb88dbb9ee990d2cd9aa9e28526c93e794fba"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:91ec6fe47b5eb5a9968c79ad9ed78c342b1f97a091677ba0e012701add857829"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cb841572862f629b99725ebaec3287fc6d275be9b14443ea746c1dd325053cbd"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c380b27d041209b849ed246b111b7c166ba36d7933ec6e41175fd15ab9eb1572"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c9af5a3b406a50e313467e3565fc99929717f780164fe6fbb7704edba0cebbe"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5671583eab84af046a397d6d0ba25343c00cd50bce03787948e0fff01d4fd9b1"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:84a6f19ce086c1bf894644b43cd129702f781ba5751ca8572f08aa40ef0ab7b7"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1e7723bd90ef94eda669a3c2c19d549874dd5badaeefabefd26053304abe5799"}, - {file = "Pillow-9.5.0.tar.gz", hash = "sha256:bf548479d336726d7a0eceb6e767e179fbde37833ae42794602631a070d630f1"}, + {file = "Pillow-10.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891"}, + {file = "Pillow-10.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d5db32e2a6ccbb3d34d87c87b432959e0db29755727afb37290e10f6e8e62614"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edf4392b77bdc81f36e92d3a07a5cd072f90253197f4a52a55a8cec48a12483b"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:520f2a520dc040512699f20fa1c363eed506e94248d71f85412b625026f6142c"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:8c11160913e3dd06c8ffdb5f233a4f254cb449f4dfc0f8f4549eda9e542c93d1"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf"}, + {file = "Pillow-10.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d5d0dae4cfd56969d23d94dc8e89fb6a217be461c69090768227beb8ed28c0a3"}, + {file = "Pillow-10.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992"}, + {file = "Pillow-10.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:dffe31a7f47b603318c609f378ebcd57f1554a3a6a8effbc59c3c69f804296de"}, + {file = "Pillow-10.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485"}, + {file = "Pillow-10.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d35e3c8d9b1268cbf5d3670285feb3528f6680420eafe35cccc686b73c1e330f"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ed64f9ca2f0a95411e88a4efbd7a29e5ce2cea36072c53dd9d26d9c76f753b3"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cd25d2a9d2b36fcb318882481367956d2cf91329f6892fe5d385c346c0649629"}, + {file = "Pillow-10.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3b08d4cc24f471b2c8ca24ec060abf4bebc6b144cb89cba638c720546b1cf538"}, + {file = "Pillow-10.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d737a602fbd82afd892ca746392401b634e278cb65d55c4b7a8f48e9ef8d008d"}, + {file = "Pillow-10.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:3a82c40d706d9aa9734289740ce26460a11aeec2d9c79b7af87bb35f0073c12f"}, + {file = "Pillow-10.0.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:d80cf684b541685fccdd84c485b31ce73fc5c9b5d7523bf1394ce134a60c6883"}, + {file = "Pillow-10.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76de421f9c326da8f43d690110f0e79fe3ad1e54be811545d7d91898b4c8493e"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81ff539a12457809666fef6624684c008e00ff6bf455b4b89fd00a140eecd640"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce543ed15570eedbb85df19b0a1a7314a9c8141a36ce089c0a894adbfccb4568"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:685ac03cc4ed5ebc15ad5c23bc555d68a87777586d970c2c3e216619a5476223"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d72e2ecc68a942e8cf9739619b7f408cc7b272b279b56b2c83c6123fcfa5cdff"}, + {file = "Pillow-10.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d50b6aec14bc737742ca96e85d6d0a5f9bfbded018264b3b70ff9d8c33485551"}, + {file = "Pillow-10.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5"}, + {file = "Pillow-10.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:f31f9fdbfecb042d046f9d91270a0ba28368a723302786c0009ee9b9f1f60199"}, + {file = "Pillow-10.0.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:349930d6e9c685c089284b013478d6f76e3a534e36ddfa912cde493f235372f3"}, + {file = "Pillow-10.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3a684105f7c32488f7153905a4e3015a3b6c7182e106fe3c37fbb5ef3e6994c3"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f07ea8d2f827d7d2a49ecf1639ec02d75ffd1b88dcc5b3a61bbb37a8759ad8d"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:f88a0b92277de8e3ca715a0d79d68dc82807457dae3ab8699c758f07c20b3c51"}, + {file = "Pillow-10.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86"}, + {file = "Pillow-10.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7"}, + {file = "Pillow-10.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:dbc02381779d412145331789b40cc7b11fdf449e5d94f6bc0b080db0a56ea3f0"}, + {file = "Pillow-10.0.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:9211e7ad69d7c9401cfc0e23d49b69ca65ddd898976d660a2fa5904e3d7a9baa"}, + {file = "Pillow-10.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:faaf07ea35355b01a35cb442dd950d8f1bb5b040a7787791a535de13db15ed90"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9f72a021fbb792ce98306ffb0c348b3c9cb967dce0f12a49aa4c3d3fdefa967"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f7c16705f44e0504a3a2a14197c1f0b32a95731d251777dcb060aa83022cb2d"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:76edb0a1fa2b4745fb0c99fb9fb98f8b180a1bbceb8be49b087e0b21867e77d3"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:368ab3dfb5f49e312231b6f27b8820c823652b7cd29cfbd34090565a015e99ba"}, + {file = "Pillow-10.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:608bfdee0d57cf297d32bcbb3c728dc1da0907519d1784962c5f0c68bb93e5a3"}, + {file = "Pillow-10.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5c6e3df6bdd396749bafd45314871b3d0af81ff935b2d188385e970052091017"}, + {file = "Pillow-10.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:7be600823e4c8631b74e4a0d38384c73f680e6105a7d3c6824fcf226c178c7e6"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:92be919bbc9f7d09f7ae343c38f5bb21c973d2576c1d45600fce4b74bafa7ac0"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8182b523b2289f7c415f589118228d30ac8c355baa2f3194ced084dac2dbba"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:38250a349b6b390ee6047a62c086d3817ac69022c127f8a5dc058c31ccef17f3"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:88af2003543cc40c80f6fca01411892ec52b11021b3dc22ec3bc9d5afd1c5334"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce7b031a6fc11365970e6a5686d7ba8c63e4c1cf1ea143811acbb524295eabed"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:db24668940f82321e746773a4bc617bfac06ec831e5c88b643f91f122a785684"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:efe8c0681042536e0d06c11f48cebe759707c9e9abf880ee213541c5b46c5bf3"}, + {file = "Pillow-10.0.0.tar.gz", hash = "sha256:9c82b5b3e043c7af0d95792d0d20ccf68f61a1fec6b3530e718b688422727396"}, ] [package.extras] @@ -1053,14 +1318,14 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa [[package]] name = "pluggy" -version = "1.0.0" +version = "1.2.0" description = "plugin and hook calling mechanisms for python" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, + {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, + {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, ] [package.extras] @@ -1086,6 +1351,26 @@ tomli = ">=1.2.2" [package.extras] poetry-plugin = ["poetry (>=1.0,<2.0)"] +[[package]] +name = "portalocker" +version = "2.7.0" +description = "Wraps the portalocker recipe for easy usage" +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "portalocker-2.7.0-py2.py3-none-any.whl", hash = "sha256:a07c5b4f3985c3cf4798369631fb7011adb498e2a46d8440efc75a8f29a0f983"}, + {file = "portalocker-2.7.0.tar.gz", hash = "sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51"}, +] + +[package.dependencies] +pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} + +[package.extras] +docs = ["sphinx (>=1.7.1)"] +redis = ["redis"] +tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)"] + [[package]] name = "py" version = "1.11.0" @@ -1124,48 +1409,48 @@ files = [ [[package]] name = "pydantic" -version = "1.10.9" +version = "1.10.11" description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e692dec4a40bfb40ca530e07805b1208c1de071a18d26af4a2a0d79015b352ca"}, - {file = "pydantic-1.10.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c52eb595db83e189419bf337b59154bdcca642ee4b2a09e5d7797e41ace783f"}, - {file = "pydantic-1.10.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:939328fd539b8d0edf244327398a667b6b140afd3bf7e347cf9813c736211896"}, - {file = "pydantic-1.10.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b48d3d634bca23b172f47f2335c617d3fcb4b3ba18481c96b7943a4c634f5c8d"}, - {file = "pydantic-1.10.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f0b7628fb8efe60fe66fd4adadd7ad2304014770cdc1f4934db41fe46cc8825f"}, - {file = "pydantic-1.10.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e1aa5c2410769ca28aa9a7841b80d9d9a1c5f223928ca8bec7e7c9a34d26b1d4"}, - {file = "pydantic-1.10.9-cp310-cp310-win_amd64.whl", hash = "sha256:eec39224b2b2e861259d6f3c8b6290d4e0fbdce147adb797484a42278a1a486f"}, - {file = "pydantic-1.10.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d111a21bbbfd85c17248130deac02bbd9b5e20b303338e0dbe0faa78330e37e0"}, - {file = "pydantic-1.10.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e9aec8627a1a6823fc62fb96480abe3eb10168fd0d859ee3d3b395105ae19a7"}, - {file = "pydantic-1.10.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07293ab08e7b4d3c9d7de4949a0ea571f11e4557d19ea24dd3ae0c524c0c334d"}, - {file = "pydantic-1.10.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ee829b86ce984261d99ff2fd6e88f2230068d96c2a582f29583ed602ef3fc2c"}, - {file = "pydantic-1.10.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4b466a23009ff5cdd7076eb56aca537c745ca491293cc38e72bf1e0e00de5b91"}, - {file = "pydantic-1.10.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7847ca62e581e6088d9000f3c497267868ca2fa89432714e21a4fb33a04d52e8"}, - {file = "pydantic-1.10.9-cp311-cp311-win_amd64.whl", hash = "sha256:7845b31959468bc5b78d7b95ec52fe5be32b55d0d09983a877cca6aedc51068f"}, - {file = "pydantic-1.10.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:517a681919bf880ce1dac7e5bc0c3af1e58ba118fd774da2ffcd93c5f96eaece"}, - {file = "pydantic-1.10.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67195274fd27780f15c4c372f4ba9a5c02dad6d50647b917b6a92bf00b3d301a"}, - {file = "pydantic-1.10.9-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2196c06484da2b3fded1ab6dbe182bdabeb09f6318b7fdc412609ee2b564c49a"}, - {file = "pydantic-1.10.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6257bb45ad78abacda13f15bde5886efd6bf549dd71085e64b8dcf9919c38b60"}, - {file = "pydantic-1.10.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3283b574b01e8dbc982080d8287c968489d25329a463b29a90d4157de4f2baaf"}, - {file = "pydantic-1.10.9-cp37-cp37m-win_amd64.whl", hash = "sha256:5f8bbaf4013b9a50e8100333cc4e3fa2f81214033e05ac5aa44fa24a98670a29"}, - {file = "pydantic-1.10.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9cd67fb763248cbe38f0593cd8611bfe4b8ad82acb3bdf2b0898c23415a1f82"}, - {file = "pydantic-1.10.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f50e1764ce9353be67267e7fd0da08349397c7db17a562ad036aa7c8f4adfdb6"}, - {file = "pydantic-1.10.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73ef93e5e1d3c8e83f1ff2e7fdd026d9e063c7e089394869a6e2985696693766"}, - {file = "pydantic-1.10.9-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:128d9453d92e6e81e881dd7e2484e08d8b164da5507f62d06ceecf84bf2e21d3"}, - {file = "pydantic-1.10.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ad428e92ab68798d9326bb3e5515bc927444a3d71a93b4a2ca02a8a5d795c572"}, - {file = "pydantic-1.10.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fab81a92f42d6d525dd47ced310b0c3e10c416bbfae5d59523e63ea22f82b31e"}, - {file = "pydantic-1.10.9-cp38-cp38-win_amd64.whl", hash = "sha256:963671eda0b6ba6926d8fc759e3e10335e1dc1b71ff2a43ed2efd6996634dafb"}, - {file = "pydantic-1.10.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:970b1bdc6243ef663ba5c7e36ac9ab1f2bfecb8ad297c9824b542d41a750b298"}, - {file = "pydantic-1.10.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7e1d5290044f620f80cf1c969c542a5468f3656de47b41aa78100c5baa2b8276"}, - {file = "pydantic-1.10.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83fcff3c7df7adff880622a98022626f4f6dbce6639a88a15a3ce0f96466cb60"}, - {file = "pydantic-1.10.9-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0da48717dc9495d3a8f215e0d012599db6b8092db02acac5e0d58a65248ec5bc"}, - {file = "pydantic-1.10.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0a2aabdc73c2a5960e87c3ffebca6ccde88665616d1fd6d3db3178ef427b267a"}, - {file = "pydantic-1.10.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9863b9420d99dfa9c064042304868e8ba08e89081428a1c471858aa2af6f57c4"}, - {file = "pydantic-1.10.9-cp39-cp39-win_amd64.whl", hash = "sha256:e7c9900b43ac14110efa977be3da28931ffc74c27e96ee89fbcaaf0b0fe338e1"}, - {file = "pydantic-1.10.9-py3-none-any.whl", hash = "sha256:6cafde02f6699ce4ff643417d1a9223716ec25e228ddc3b436fe7e2d25a1f305"}, - {file = "pydantic-1.10.9.tar.gz", hash = "sha256:95c70da2cd3b6ddf3b9645ecaa8d98f3d80c606624b6d245558d202cd23ea3be"}, + {file = "pydantic-1.10.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ff44c5e89315b15ff1f7fdaf9853770b810936d6b01a7bcecaa227d2f8fe444f"}, + {file = "pydantic-1.10.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a6c098d4ab5e2d5b3984d3cb2527e2d6099d3de85630c8934efcfdc348a9760e"}, + {file = "pydantic-1.10.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16928fdc9cb273c6af00d9d5045434c39afba5f42325fb990add2c241402d151"}, + {file = "pydantic-1.10.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0588788a9a85f3e5e9ebca14211a496409cb3deca5b6971ff37c556d581854e7"}, + {file = "pydantic-1.10.11-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9baf78b31da2dc3d3f346ef18e58ec5f12f5aaa17ac517e2ffd026a92a87588"}, + {file = "pydantic-1.10.11-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:373c0840f5c2b5b1ccadd9286782852b901055998136287828731868027a724f"}, + {file = "pydantic-1.10.11-cp310-cp310-win_amd64.whl", hash = "sha256:c3339a46bbe6013ef7bdd2844679bfe500347ac5742cd4019a88312aa58a9847"}, + {file = "pydantic-1.10.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:08a6c32e1c3809fbc49debb96bf833164f3438b3696abf0fbeceb417d123e6eb"}, + {file = "pydantic-1.10.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a451ccab49971af043ec4e0d207cbc8cbe53dbf148ef9f19599024076fe9c25b"}, + {file = "pydantic-1.10.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b02d24f7b2b365fed586ed73582c20f353a4c50e4be9ba2c57ab96f8091ddae"}, + {file = "pydantic-1.10.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f34739a89260dfa420aa3cbd069fbcc794b25bbe5c0a214f8fb29e363484b66"}, + {file = "pydantic-1.10.11-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e297897eb4bebde985f72a46a7552a7556a3dd11e7f76acda0c1093e3dbcf216"}, + {file = "pydantic-1.10.11-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d185819a7a059550ecb85d5134e7d40f2565f3dd94cfd870132c5f91a89cf58c"}, + {file = "pydantic-1.10.11-cp311-cp311-win_amd64.whl", hash = "sha256:4400015f15c9b464c9db2d5d951b6a780102cfa5870f2c036d37c23b56f7fc1b"}, + {file = "pydantic-1.10.11-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2417de68290434461a266271fc57274a138510dca19982336639484c73a07af6"}, + {file = "pydantic-1.10.11-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:331c031ba1554b974c98679bd0780d89670d6fd6f53f5d70b10bdc9addee1713"}, + {file = "pydantic-1.10.11-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8268a735a14c308923e8958363e3a3404f6834bb98c11f5ab43251a4e410170c"}, + {file = "pydantic-1.10.11-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:44e51ba599c3ef227e168424e220cd3e544288c57829520dc90ea9cb190c3248"}, + {file = "pydantic-1.10.11-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d7781f1d13b19700b7949c5a639c764a077cbbdd4322ed505b449d3ca8edcb36"}, + {file = "pydantic-1.10.11-cp37-cp37m-win_amd64.whl", hash = "sha256:7522a7666157aa22b812ce14c827574ddccc94f361237ca6ea8bb0d5c38f1629"}, + {file = "pydantic-1.10.11-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc64eab9b19cd794a380179ac0e6752335e9555d214cfcb755820333c0784cb3"}, + {file = "pydantic-1.10.11-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8dc77064471780262b6a68fe67e013298d130414d5aaf9b562c33987dbd2cf4f"}, + {file = "pydantic-1.10.11-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe429898f2c9dd209bd0632a606bddc06f8bce081bbd03d1c775a45886e2c1cb"}, + {file = "pydantic-1.10.11-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:192c608ad002a748e4a0bed2ddbcd98f9b56df50a7c24d9a931a8c5dd053bd3d"}, + {file = "pydantic-1.10.11-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ef55392ec4bb5721f4ded1096241e4b7151ba6d50a50a80a2526c854f42e6a2f"}, + {file = "pydantic-1.10.11-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:41e0bb6efe86281623abbeeb0be64eab740c865388ee934cd3e6a358784aca6e"}, + {file = "pydantic-1.10.11-cp38-cp38-win_amd64.whl", hash = "sha256:265a60da42f9f27e0b1014eab8acd3e53bd0bad5c5b4884e98a55f8f596b2c19"}, + {file = "pydantic-1.10.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:469adf96c8e2c2bbfa655fc7735a2a82f4c543d9fee97bd113a7fb509bf5e622"}, + {file = "pydantic-1.10.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e6cbfbd010b14c8a905a7b10f9fe090068d1744d46f9e0c021db28daeb8b6de1"}, + {file = "pydantic-1.10.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abade85268cc92dff86d6effcd917893130f0ff516f3d637f50dadc22ae93999"}, + {file = "pydantic-1.10.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9738b0f2e6c70f44ee0de53f2089d6002b10c33264abee07bdb5c7f03038303"}, + {file = "pydantic-1.10.11-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:787cf23e5a0cde753f2eabac1b2e73ae3844eb873fd1f5bdbff3048d8dbb7604"}, + {file = "pydantic-1.10.11-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:174899023337b9fc685ac8adaa7b047050616136ccd30e9070627c1aaab53a13"}, + {file = "pydantic-1.10.11-cp39-cp39-win_amd64.whl", hash = "sha256:1954f8778489a04b245a1e7b8b22a9d3ea8ef49337285693cf6959e4b757535e"}, + {file = "pydantic-1.10.11-py3-none-any.whl", hash = "sha256:008c5e266c8aada206d0627a011504e14268a62091450210eda7c07fabe6963e"}, + {file = "pydantic-1.10.11.tar.gz", hash = "sha256:f66d479cf7eb331372c470614be6511eae96f1f120344c25f3f9bb59fb1b5528"}, ] [package.dependencies] @@ -1175,6 +1460,21 @@ typing-extensions = ">=4.2.0" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "pydot" +version = "1.4.2" +description = "Python interface to Graphviz's Dot" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pydot-1.4.2-py2.py3-none-any.whl", hash = "sha256:66c98190c65b8d2e2382a441b4c0edfdb4f4c025ef9cb9874de478fb0793a451"}, + {file = "pydot-1.4.2.tar.gz", hash = "sha256:248081a39bcb56784deb018977e428605c1c758f10897a339fce1dd728ff007d"}, +] + +[package.dependencies] +pyparsing = ">=2.1.4" + [[package]] name = "pyexasol" version = "0.25.2" @@ -1200,6 +1500,33 @@ pandas = ["pandas"] rapidjson = ["python-rapidjson"] ujson = ["ujson"] +[[package]] +name = "pynacl" +version = "1.5.0" +description = "Python binding to the Networking and Cryptography (NaCl) library" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"}, + {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"}, +] + +[package.dependencies] +cffi = ">=1.4.1" + +[package.extras] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] + [[package]] name = "pyopenssl" version = "23.2.0" @@ -1234,16 +1561,28 @@ files = [ [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pyreadline3" +version = "3.4.1" +description = "A python implementation of GNU readline." +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "pyreadline3-3.4.1-py3-none-any.whl", hash = "sha256:b0efb6516fd4fb07b45949053826a62fa4cb353db5be2bbb4a7aa1fdd1e345fb"}, + {file = "pyreadline3-3.4.1.tar.gz", hash = "sha256:6f3d1f7b8a31ba32b73917cefc1f28cc660562f39aea8646d30bd6eff21f7bae"}, +] + [[package]] name = "pytest" -version = "7.3.2" +version = "7.4.0" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.3.2-py3-none-any.whl", hash = "sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295"}, - {file = "pytest-7.3.2.tar.gz", hash = "sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b"}, + {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, + {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, ] [package.dependencies] @@ -1307,6 +1646,27 @@ files = [ [package.dependencies] pytest = ">=3.6" +[[package]] +name = "python-daemon" +version = "3.0.1" +description = "Library to implement a well-behaved Unix daemon process." +category = "dev" +optional = false +python-versions = ">=3" +files = [ + {file = "python-daemon-3.0.1.tar.gz", hash = "sha256:6c57452372f7eaff40934a1c03ad1826bf5e793558e87fef49131e6464b4dae5"}, + {file = "python_daemon-3.0.1-py3-none-any.whl", hash = "sha256:42bb848a3260a027fa71ad47ecd959e471327cb34da5965962edd5926229f341"}, +] + +[package.dependencies] +docutils = "*" +lockfile = ">=0.10" +setuptools = ">=62.4.0" + +[package.extras] +devel = ["coverage", "docutils", "isort", "testscenarios (>=0.4)", "testtools", "twine"] +test = ["coverage", "docutils", "testscenarios (>=0.4)", "testtools"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -1334,6 +1694,30 @@ files = [ {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, ] +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + [[package]] name = "pyzmq" version = "24.0.1" @@ -1461,46 +1845,46 @@ pyasn1 = ">=0.1.3" [[package]] name = "scikit-learn" -version = "1.2.2" +version = "1.3.0" description = "A set of python modules for machine learning and data mining" category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "scikit-learn-1.2.2.tar.gz", hash = "sha256:8429aea30ec24e7a8c7ed8a3fa6213adf3814a6efbea09e16e0a0c71e1a1a3d7"}, - {file = "scikit_learn-1.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99cc01184e347de485bf253d19fcb3b1a3fb0ee4cea5ee3c43ec0cc429b6d29f"}, - {file = "scikit_learn-1.2.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e6e574db9914afcb4e11ade84fab084536a895ca60aadea3041e85b8ac963edb"}, - {file = "scikit_learn-1.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fe83b676f407f00afa388dd1fdd49e5c6612e551ed84f3b1b182858f09e987d"}, - {file = "scikit_learn-1.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e2642baa0ad1e8f8188917423dd73994bf25429f8893ddbe115be3ca3183584"}, - {file = "scikit_learn-1.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:ad66c3848c0a1ec13464b2a95d0a484fd5b02ce74268eaa7e0c697b904f31d6c"}, - {file = "scikit_learn-1.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dfeaf8be72117eb61a164ea6fc8afb6dfe08c6f90365bde2dc16456e4bc8e45f"}, - {file = "scikit_learn-1.2.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:fe0aa1a7029ed3e1dcbf4a5bc675aa3b1bc468d9012ecf6c6f081251ca47f590"}, - {file = "scikit_learn-1.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:065e9673e24e0dc5113e2dd2b4ca30c9d8aa2fa90f4c0597241c93b63130d233"}, - {file = "scikit_learn-1.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf036ea7ef66115e0d49655f16febfa547886deba20149555a41d28f56fd6d3c"}, - {file = "scikit_learn-1.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:8b0670d4224a3c2d596fd572fb4fa673b2a0ccfb07152688ebd2ea0b8c61025c"}, - {file = "scikit_learn-1.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9c710ff9f9936ba8a3b74a455ccf0dcf59b230caa1e9ba0223773c490cab1e51"}, - {file = "scikit_learn-1.2.2-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:2dd3ffd3950e3d6c0c0ef9033a9b9b32d910c61bd06cb8206303fb4514b88a49"}, - {file = "scikit_learn-1.2.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44b47a305190c28dd8dd73fc9445f802b6ea716669cfc22ab1eb97b335d238b1"}, - {file = "scikit_learn-1.2.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:953236889928d104c2ef14027539f5f2609a47ebf716b8cbe4437e85dce42744"}, - {file = "scikit_learn-1.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:7f69313884e8eb311460cc2f28676d5e400bd929841a2c8eb8742ae78ebf7c20"}, - {file = "scikit_learn-1.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8156db41e1c39c69aa2d8599ab7577af53e9e5e7a57b0504e116cc73c39138dd"}, - {file = "scikit_learn-1.2.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:fe175ee1dab589d2e1033657c5b6bec92a8a3b69103e3dd361b58014729975c3"}, - {file = "scikit_learn-1.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d5312d9674bed14f73773d2acf15a3272639b981e60b72c9b190a0cffed5bad"}, - {file = "scikit_learn-1.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea061bf0283bf9a9f36ea3c5d3231ba2176221bbd430abd2603b1c3b2ed85c89"}, - {file = "scikit_learn-1.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:6477eed40dbce190f9f9e9d0d37e020815825b300121307942ec2110302b66a3"}, + {file = "scikit-learn-1.3.0.tar.gz", hash = "sha256:8be549886f5eda46436b6e555b0e4873b4f10aa21c07df45c4bc1735afbccd7a"}, + {file = "scikit_learn-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:981287869e576d42c682cf7ca96af0c6ac544ed9316328fd0d9292795c742cf5"}, + {file = "scikit_learn-1.3.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:436aaaae2c916ad16631142488e4c82f4296af2404f480e031d866863425d2a2"}, + {file = "scikit_learn-1.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7e28d8fa47a0b30ae1bd7a079519dd852764e31708a7804da6cb6f8b36e3630"}, + {file = "scikit_learn-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae80c08834a473d08a204d966982a62e11c976228d306a2648c575e3ead12111"}, + {file = "scikit_learn-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:552fd1b6ee22900cf1780d7386a554bb96949e9a359999177cf30211e6b20df6"}, + {file = "scikit_learn-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79970a6d759eb00a62266a31e2637d07d2d28446fca8079cf9afa7c07b0427f8"}, + {file = "scikit_learn-1.3.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:850a00b559e636b23901aabbe79b73dc604b4e4248ba9e2d6e72f95063765603"}, + {file = "scikit_learn-1.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee04835fb016e8062ee9fe9074aef9b82e430504e420bff51e3e5fffe72750ca"}, + {file = "scikit_learn-1.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d953531f5d9f00c90c34fa3b7d7cfb43ecff4c605dac9e4255a20b114a27369"}, + {file = "scikit_learn-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:151ac2bf65ccf363664a689b8beafc9e6aae36263db114b4ca06fbbbf827444a"}, + {file = "scikit_learn-1.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a885a9edc9c0a341cab27ec4f8a6c58b35f3d449c9d2503a6fd23e06bbd4f6a"}, + {file = "scikit_learn-1.3.0-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:9877af9c6d1b15486e18a94101b742e9d0d2f343d35a634e337411ddb57783f3"}, + {file = "scikit_learn-1.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c470f53cea065ff3d588050955c492793bb50c19a92923490d18fcb637f6383a"}, + {file = "scikit_learn-1.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6e2d7389542eae01077a1ee0318c4fec20c66c957f45c7aac0c6eb0fe3c612"}, + {file = "scikit_learn-1.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:3a11936adbc379a6061ea32fa03338d4ca7248d86dd507c81e13af428a5bc1db"}, + {file = "scikit_learn-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:998d38fcec96584deee1e79cd127469b3ad6fefd1ea6c2dfc54e8db367eb396b"}, + {file = "scikit_learn-1.3.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:ded35e810438a527e17623ac6deae3b360134345b7c598175ab7741720d7ffa7"}, + {file = "scikit_learn-1.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e8102d5036e28d08ab47166b48c8d5e5810704daecf3a476a4282d562be9a28"}, + {file = "scikit_learn-1.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7617164951c422747e7c32be4afa15d75ad8044f42e7d70d3e2e0429a50e6718"}, + {file = "scikit_learn-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:1d54fb9e6038284548072df22fd34777e434153f7ffac72c8596f2d6987110dd"}, ] [package.dependencies] joblib = ">=1.1.1" numpy = ">=1.17.3" -scipy = ">=1.3.2" +scipy = ">=1.5.0" threadpoolctl = ">=2.0.0" [package.extras] benchmark = ["matplotlib (>=3.1.3)", "memory-profiler (>=0.57.0)", "pandas (>=1.0.5)"] -docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.1.3)", "memory-profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.0.5)", "plotly (>=5.10.0)", "pooch (>=1.6.0)", "scikit-image (>=0.16.2)", "seaborn (>=0.9.0)", "sphinx (>=4.0.1)", "sphinx-gallery (>=0.7.0)", "sphinx-prompt (>=1.3.0)", "sphinxext-opengraph (>=0.4.2)"] -examples = ["matplotlib (>=3.1.3)", "pandas (>=1.0.5)", "plotly (>=5.10.0)", "pooch (>=1.6.0)", "scikit-image (>=0.16.2)", "seaborn (>=0.9.0)"] -tests = ["black (>=22.3.0)", "flake8 (>=3.8.2)", "matplotlib (>=3.1.3)", "mypy (>=0.961)", "numpydoc (>=1.2.0)", "pandas (>=1.0.5)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pytest (>=5.3.1)", "pytest-cov (>=2.9.0)", "scikit-image (>=0.16.2)"] +docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.1.3)", "memory-profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.0.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.16.2)", "seaborn (>=0.9.0)", "sphinx (>=6.0.0)", "sphinx-copybutton (>=0.5.2)", "sphinx-gallery (>=0.10.1)", "sphinx-prompt (>=1.3.0)", "sphinxext-opengraph (>=0.4.2)"] +examples = ["matplotlib (>=3.1.3)", "pandas (>=1.0.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.16.2)", "seaborn (>=0.9.0)"] +tests = ["black (>=23.3.0)", "matplotlib (>=3.1.3)", "mypy (>=1.3)", "numpydoc (>=1.2.0)", "pandas (>=1.0.5)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.0.272)", "scikit-image (>=0.16.2)"] [[package]] name = "scipy" @@ -1665,6 +2049,18 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "smmap" +version = "5.0.0" +description = "A pure Python implementation of a sliding window memory map manager" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, + {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -1677,6 +2073,18 @@ files = [ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] +[[package]] +name = "stopwatch-py" +version = "2.0.1" +description = "A simple stopwatch for python" +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "stopwatch.py-2.0.1-py3-none-any.whl", hash = "sha256:5802a0178d766120c11dd5df8ae838e9beccb8c88329dbd5f0f7ac4b7fed9107"}, + {file = "stopwatch.py-2.0.1.tar.gz", hash = "sha256:8cc94ba0f6469d434eabd8b227166e595fd42350e7f66dbf1a1a80697f60cc79"}, +] + [[package]] name = "structlog" version = "22.3.0" @@ -1734,6 +2142,27 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tornado" +version = "6.3.2" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +category = "dev" +optional = false +python-versions = ">= 3.8" +files = [ + {file = "tornado-6.3.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829"}, + {file = "tornado-6.3.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b46a6ab20f5c7c1cb949c72c1994a4585d2eaa0be4853f50a03b5031e964fc7c"}, + {file = "tornado-6.3.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f"}, + {file = "tornado-6.3.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05615096845cf50a895026f749195bf0b10b8909f9be672f50b0fe69cba368e4"}, + {file = "tornado-6.3.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b17b1cf5f8354efa3d37c6e28fdfd9c1c1e5122f2cb56dac121ac61baa47cbe"}, + {file = "tornado-6.3.2-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:29e71c847a35f6e10ca3b5c2990a52ce38b233019d8e858b755ea6ce4dcdd19d"}, + {file = "tornado-6.3.2-cp38-abi3-musllinux_1_1_i686.whl", hash = "sha256:834ae7540ad3a83199a8da8f9f2d383e3c3d5130a328889e4cc991acc81e87a0"}, + {file = "tornado-6.3.2-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6a0848f1aea0d196a7c4f6772197cbe2abc4266f836b0aac76947872cd29b411"}, + {file = "tornado-6.3.2-cp38-abi3-win32.whl", hash = "sha256:7efcbcc30b7c654eb6a8c9c9da787a851c18f8ccd4a5a3a95b05c7accfa068d2"}, + {file = "tornado-6.3.2-cp38-abi3-win_amd64.whl", hash = "sha256:0c325e66c8123c606eea33084976c832aa4e766b7dff8aedd7587ea44a604cdf"}, + {file = "tornado-6.3.2.tar.gz", hash = "sha256:4b927c4f19b71e627b13f3db2324e4ae660527143f9e1f2e2fb404f3a187e2ba"}, +] + [[package]] name = "typeguard" version = "2.13.3" @@ -1752,14 +2181,14 @@ test = ["mypy", "pytest", "typing-extensions"] [[package]] name = "typing-extensions" -version = "4.6.3" +version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.6.3-py3-none-any.whl", hash = "sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26"}, - {file = "typing_extensions-4.6.3.tar.gz", hash = "sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5"}, + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] [[package]] @@ -1782,14 +2211,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "websocket-client" -version = "1.6.0" +version = "1.6.1" description = "WebSocket client for Python with low level API options" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "websocket-client-1.6.0.tar.gz", hash = "sha256:e84c7eafc66aade6d1967a51dfd219aabdf81d15b9705196e11fd81f48666b78"}, - {file = "websocket_client-1.6.0-py3-none-any.whl", hash = "sha256:72d7802608745b0a212f79b478642473bd825777d8637b6c8c421bf167790d4f"}, + {file = "websocket-client-1.6.1.tar.gz", hash = "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd"}, + {file = "websocket_client-1.6.1-py3-none-any.whl", hash = "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"}, ] [package.extras] @@ -1816,4 +2245,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.8,<4.0" -content-hash = "4daebba8cce2bfcd59cb5145b79d0e2b67f42a6d801adcac8a67ad97f59e6d44" +content-hash = "9bddea4f4be7886cc47ac856cd333dce46192700135b58e115f61282313774c1" diff --git a/pyproject.toml b/pyproject.toml index 675451c1..4043e6c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ poethepoet = "^0.13.0" pytest-assume = "^2.4.3" exasol-udf-mock-python = { git = "https://github.com/exasol/udf-mock-python.git", branch = "main" } pytest-cov = "^3.0.0" +exasol-integration-test-docker-environment = "1.7.1" [tool.poetry.group.dev.dependencies] pytest-repeat = "^0.9.1" diff --git a/scripts/start_integration_test_environment.sh b/scripts/start_integration_test_environment.sh index 7f37cfbb..aeb7f044 100755 --- a/scripts/start_integration_test_environment.sh +++ b/scripts/start_integration_test_environment.sh @@ -2,19 +2,4 @@ set -euo pipefail -GIT_REF=main -CHECKOUT_PATH=/tmp/integration-test-docker-environment - -if [ -d "$CHECKOUT_PATH" ] -then - cd "$CHECKOUT_PATH" - git checkout "$GIT_REF" - git pull -else - git clone https://github.com/exasol/integration-test-docker-environment.git "$CHECKOUT_PATH" - cd "$CHECKOUT_PATH" - git checkout "$GIT_REF" -fi - - -./start-test-env spawn-test-environment --environment-name test --database-port-forward 9563 --bucketfs-port-forward 6666 --db-mem-size 4GB --nameserver 8.8.8.8 +./scripts/run_in_dev_env.sh poetry run itde spawn-test-environment --environment-name test --database-port-forward 9563 --bucketfs-port-forward 6666 --db-mem-size 4GB --nameserver 8.8.8.8 diff --git a/tests/udf_communication/peer_communication/analyze_log.py b/tests/udf_communication/peer_communication/analyze_log.py new file mode 100644 index 00000000..b8a294d1 --- /dev/null +++ b/tests/udf_communication/peer_communication/analyze_log.py @@ -0,0 +1,117 @@ +import json +from collections import defaultdict, Counter +from pathlib import Path +from typing import Dict, List, Callable + + +def is_log_sequence_ok(lines: List[Dict[str, str]], line_predicate: Callable[[Dict[str, str]], bool]): + result = False + for line in lines: + if line_predicate(line): + result = True + return result + + +def is_peer_ready(line: Dict[str, str]): + return line["module"] == "peer_is_ready_sender" and line["event"] == "send" + + +def is_connection_acknowledged(line: Dict[str, str]): + return line["module"] == "background_peer_state" and line["event"] == "received_acknowledge_connection" + + +def is_connection_synchronized(line: Dict[str, str]): + return line["module"] == "background_peer_state" and line["event"] == "received_synchronize_connection" + + +def analyze_source_target_interaction(): + print("analyze_source_target_interaction") + root = Path(__file__).parent + with open(root / "test_add_peer.log") as f: + group_source_target_map = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) + collect_source_target_interaction(f, group_source_target_map) + print_source_target_interaction(group_source_target_map) + + +def print_source_target_interaction(group_source_target_map): + predicates = { + "is_peer_ready": is_peer_ready, + "is_connection_acknowledged": is_connection_acknowledged, + "is_connection_synchronized": is_connection_synchronized + } + for group, sources in group_source_target_map.items(): + ok = Counter() + not_ok = Counter() + for source, targets in sources.items(): + for target, lines in targets.items(): + for predicate_name, predicate in predicates.items(): + if not is_log_sequence_ok(lines, predicate): + print(f"========== {predicate_name}-{group}-{source}-{target} ============") + not_ok.update((predicate_name,)) + else: + ok.update((predicate_name,)) + for predicate_name in predicates.keys(): + print(f"{group} {predicate_name} ok {ok[predicate_name]} not_ok {not_ok[predicate_name]}") + + +def collect_source_target_interaction(f, group_source_target_map): + for line in iter(f.readline, ""): + json_line = json.loads(line) + if ("peer" in json_line + and "my_connection_info" in json_line + and "event" in json_line + and "module" in json_line + and json_line["event"] != "send_if_necessary" + ): + group = json_line["my_connection_info"]["group_identifier"] + source = json_line["my_connection_info"]["name"] + target = json_line["peer"]["connection_info"]["name"] + group_source_target_map[group][source][target].append({ + "event": json_line["event"], + "module": json_line["module"], + "timestamp": json_line["timestamp"], + }) + + +def collect_close(f, group_source_map): + for line in iter(f.readline, ""): + json_line = json.loads(line) + if ("name" in json_line + and "group_identifier" in json_line + and "event" in json_line + and json_line["event"].startswith("after") + ): + group = json_line["group_identifier"] + source = json_line["name"] + group_source_map[group][source].append({ + "timestamp": json_line["timestamp"], + "event": json_line["event"] + }) + + +def print_close(group_source_map): + for group, sources in group_source_map.items(): + for source, lines in sources.items(): + # There are two steps for closing a test. + # First, closing the PeerCommunicator. + # Second, closing the zmq context. + # If we have more or less, something is off. + if len(lines) != 2: + print(f"============== {group}-{source} ===============") + for line in lines: + print(line) + print(f"{group} after ... {len(sources)}") + + +def analyze_close(): + print("analyze_close") + root = Path(__file__).parent + with open(root / "test_add_peer.log") as f: + group_source_map = defaultdict(lambda: defaultdict(list)) + collect_close(f, group_source_map) + print_close(group_source_map) + + +if __name__ == "__main__": + analyze_source_target_interaction() + analyze_close() diff --git a/tests/udf_communication/peer_communication/conditional_method_dropper.py b/tests/udf_communication/peer_communication/conditional_method_dropper.py new file mode 100644 index 00000000..43eb4c12 --- /dev/null +++ b/tests/udf_communication/peer_communication/conditional_method_dropper.py @@ -0,0 +1,12 @@ +from structlog import DropEvent + + +class ConditionalMethodDropper: + def __init__(self, method_name): + self._method_name = method_name + + def __call__(self, logger, method_name, event_dict): + if method_name == self._method_name: + raise DropEvent + + return event_dict diff --git a/tests/udf_communication/peer_communication/mock_cast.py b/tests/udf_communication/peer_communication/mock_cast.py new file mode 100644 index 00000000..3a3866ea --- /dev/null +++ b/tests/udf_communication/peer_communication/mock_cast.py @@ -0,0 +1,6 @@ +from typing import Any, cast +from unittest.mock import Mock + + +def mock_cast(obj: Any) -> Mock: + return cast(Mock, obj) diff --git a/tests/udf_communication/peer_communication/test_abort_timeout_sender.py b/tests/udf_communication/peer_communication/test_abort_timeout_sender.py new file mode 100644 index 00000000..6799828a --- /dev/null +++ b/tests/udf_communication/peer_communication/test_abort_timeout_sender.py @@ -0,0 +1,168 @@ +import dataclasses +from typing import Union, cast, Any +from unittest.mock import MagicMock, Mock, create_autospec, call + +from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo +from exasol_advanced_analytics_framework.udf_communication.ip_address import IPAddress, Port +from exasol_advanced_analytics_framework.udf_communication.messages import TimeoutMessage +from exasol_advanced_analytics_framework.udf_communication.peer import Peer +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.abort_timeout_sender import \ + AbortTimeoutSender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.timer import Timer +from exasol_advanced_analytics_framework.udf_communication.serialization import serialize_message +from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import Socket + + +def mock_cast(obj: Any) -> Mock: + return cast(Mock, obj) + + +@dataclasses.dataclass() +class TestSetup: + timer_mock: Union[MagicMock, Timer] + out_control_socket_mock: Union[MagicMock, Socket] + abort_timeout_sender: AbortTimeoutSender = None + + def reset_mock(self): + self.out_control_socket_mock.reset_mock() + self.timer_mock.reset_mock() + + +def create_test_setup(): + peer = Peer( + connection_info=ConnectionInfo( + name="t2", + ipaddress=IPAddress(ip_address="127.0.0.1"), + port=Port(port=12), + group_identifier="g" + )) + my_connection_info = ConnectionInfo( + name="t1", + ipaddress=IPAddress(ip_address="127.0.0.1"), + port=Port(port=11), + group_identifier="g" + ) + timer_mock = create_autospec(Timer) + out_control_socket_mock = create_autospec(Socket) + abort_timeout_sender = AbortTimeoutSender( + peer=peer, + my_connection_info=my_connection_info, + out_control_socket=out_control_socket_mock, + timer=timer_mock + ) + return TestSetup( + timer_mock=timer_mock, + out_control_socket_mock=out_control_socket_mock, + abort_timeout_sender=abort_timeout_sender + ) + + +def test_init(): + test_setup = create_test_setup() + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [] + ) + + +def test_send_if_necessary_after_init_and_is_time_false(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = False + test_setup.reset_mock() + + test_setup.abort_timeout_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [ + call.is_time() + ] + ) + + +def test_send_if_necessary_after_init_and_is_time_true(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.reset_mock() + + test_setup.abort_timeout_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == + [ + call.send(serialize_message(TimeoutMessage())) + ] + and test_setup.timer_mock.mock_calls == [ + call.is_time() + ] + ) + + +def test_send_if_necessary_twice_and_is_time_false(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = False + test_setup.abort_timeout_sender.send_if_necessary() + test_setup.reset_mock() + + test_setup.abort_timeout_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [ + call.is_time() + ] + ) + + +def test_send_if_necessary_twice_and_is_time_true(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.abort_timeout_sender.send_if_necessary() + test_setup.reset_mock() + + test_setup.abort_timeout_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [ + call.is_time() + ] + ) + + +def test_send_if_necessary_after_stop_and_is_time_false(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = False + test_setup.abort_timeout_sender.stop() + test_setup.reset_mock() + + test_setup.abort_timeout_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [call.is_time()] + ) + + +def test_send_if_necessary_after_stop_and_is_time_true(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.abort_timeout_sender.stop() + test_setup.reset_mock() + + test_setup.abort_timeout_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [call.is_time()] + ) + + +def test_reset_timer(): + test_setup = create_test_setup() + print(test_setup.timer_mock.mock_calls) + test_setup.abort_timeout_sender.reset_timer() + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [call.reset_timer()] + ) diff --git a/tests/udf_communication/peer_communication/test_add_peer.py b/tests/udf_communication/peer_communication/test_add_peer.py index 9d67514c..3851705e 100644 --- a/tests/udf_communication/peer_communication/test_add_peer.py +++ b/tests/udf_communication/peer_communication/test_add_peer.py @@ -1,3 +1,4 @@ +import sys import time import traceback from pathlib import Path @@ -8,6 +9,7 @@ import zmq from numpy.random import RandomState from structlog import WriteLoggerFactory +from structlog.tracebacks import ExceptionDictTransformer from structlog.types import FilteringBoundLogger from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo @@ -18,6 +20,7 @@ from exasol_advanced_analytics_framework.udf_communication.socket_factory.fault_injection import \ FaultInjectionSocketFactory from exasol_advanced_analytics_framework.udf_communication.socket_factory.zmq_wrapper import ZMQSocketFactory +from tests.udf_communication.peer_communication.conditional_method_dropper import ConditionalMethodDropper from tests.udf_communication.peer_communication.utils import TestProcess, BidirectionalQueue, assert_processes_finish structlog.configure( @@ -25,15 +28,16 @@ logger_factory=WriteLoggerFactory(file=Path(__file__).with_suffix(".log").open("wt")), processors=[ structlog.contextvars.merge_contextvars, + ConditionalMethodDropper(method_name="debug"), structlog.processors.add_log_level, - structlog.processors.StackInfoRenderer(), - structlog.dev.set_exc_info, structlog.processors.TimeStamper(), + structlog.processors.ExceptionRenderer(exception_formatter=ExceptionDictTransformer(locals_max_string=320)), + structlog.processors.CallsiteParameterAdder(), structlog.processors.JSONRenderer() ] ) -LOGGER: FilteringBoundLogger = structlog.get_logger().bind(module_name=__name__) +LOGGER: FilteringBoundLogger = structlog.get_logger() def run(name: str, group_identifier: str, number_of_instances: int, queue: BidirectionalQueue, seed: int): @@ -42,7 +46,7 @@ def run(name: str, group_identifier: str, number_of_instances: int, queue: Bidir listen_ip = IPAddress(ip_address=f"127.1.0.1") context = zmq.Context() socket_factory = ZMQSocketFactory(context) - socket_factory = FaultInjectionSocketFactory(socket_factory, 0.0, RandomState(seed)) + socket_factory = FaultInjectionSocketFactory(socket_factory, 0.01, RandomState(seed)) com = PeerCommunicator( name=name, number_of_peers=number_of_instances, @@ -55,30 +59,57 @@ def run(name: str, group_identifier: str, number_of_instances: int, queue: Bidir for index, connection_info in peer_connection_infos.items(): com.register_peer(connection_info) peers = com.peers(timeout_in_milliseconds=None) - logger.info("peers", peers=peers) + logger.info("peers", number_of_peers=len(peers)) queue.put(peers) finally: com.close() + logger.info("after close") + context.destroy(linger=0) + logger.info("after destroy") + for frame in sys._current_frames().values(): + stacktrace = traceback.format_stack(frame) + logger.info("Frame", stacktrace=stacktrace) except Exception as e: - logger.exception("Exception during test", stacktrace=traceback.format_exc()) + queue.put([]) + logger.exception("Exception during test") @pytest.mark.parametrize("number_of_instances, repetitions", [(2, 1000), (10, 100)]) def test_reliability(number_of_instances: int, repetitions: int): + run_test_with_repetitions(number_of_instances, repetitions) + + +REPETITIONS_FOR_FUNCTIONALITY = 2 + + +def test_functionality_2(): + run_test_with_repetitions(2, REPETITIONS_FOR_FUNCTIONALITY) + + +def test_functionality_10(): + run_test_with_repetitions(10, REPETITIONS_FOR_FUNCTIONALITY) + + +def test_functionality_25(): + run_test_with_repetitions(25, REPETITIONS_FOR_FUNCTIONALITY) + + +def run_test_with_repetitions(number_of_instances: int, repetitions: int): for i in range(repetitions): + LOGGER.info(f"Start iteration", + iteration=i + 1, + repetitions=repetitions, + number_of_instances=number_of_instances) + start_time = time.monotonic() group = f"{time.monotonic_ns()}" expected_peers_of_threads, peers_of_threads = run_test(group, number_of_instances, seed=i) assert expected_peers_of_threads == peers_of_threads - - -def test_functionality(): - group = f"{time.monotonic_ns()}" - logger = LOGGER.bind(group=group, location="test") - logger.info("start") - number_of_instances = 2 - expected_peers_of_threads, peers_of_threads = run_test(group, number_of_instances, 0) - assert expected_peers_of_threads == peers_of_threads - logger.info("success") + end_time = time.monotonic() + LOGGER.info(f"Finish iteration", + iteration=i + 1, + repetitions=repetitions, + number_of_instances=number_of_instances, + duration=end_time - start_time) def run_test(group: str, number_of_instances: int, seed: int): @@ -90,7 +121,7 @@ def run_test(group: str, number_of_instances: int, seed: int): connection_infos[i] = processes[i].get() for i in range(number_of_instances): t = processes[i].put(connection_infos) - assert_processes_finish(processes, timeout_in_seconds=240) + assert_processes_finish(processes, timeout_in_seconds=180) peers_of_threads: Dict[int, List[ConnectionInfo]] = {} for i in range(number_of_instances): peers_of_threads[i] = processes[i].get() diff --git a/tests/udf_communication/peer_communication/test_background_peer_state.py b/tests/udf_communication/peer_communication/test_background_peer_state.py new file mode 100644 index 00000000..ac5bed5a --- /dev/null +++ b/tests/udf_communication/peer_communication/test_background_peer_state.py @@ -0,0 +1,149 @@ +import dataclasses +from typing import Union, cast, Any +from unittest.mock import MagicMock, Mock, create_autospec, call + +from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import Socket, \ + SocketFactory, SocketType + +from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo +from exasol_advanced_analytics_framework.udf_communication.ip_address import IPAddress, Port +from exasol_advanced_analytics_framework.udf_communication.messages import AcknowledgeConnectionMessage, Message +from exasol_advanced_analytics_framework.udf_communication.peer import Peer +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.abort_timeout_sender import \ + AbortTimeoutSender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.background_peer_state import \ + BackgroundPeerState +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.peer_is_ready_sender import \ + PeerIsReadySender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.sender import Sender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.synchronize_connection_sender import \ + SynchronizeConnectionSender + + +def mock_cast(obj: Any) -> Mock: + return cast(Mock, obj) + + +@dataclasses.dataclass() +class TestSetup: + peer: Peer + my_connection_info: ConnectionInfo + socket_factory_mock: Union[MagicMock, SocketFactory] + receive_socket_mock: Union[MagicMock, Socket] + sender_mock: Union[MagicMock, Sender] + abort_timeout_sender_mock: Union[MagicMock, AbortTimeoutSender] + peer_is_ready_sender_mock: Union[MagicMock, PeerIsReadySender] + synchronize_connection_sender_mock: Union[MagicMock, SynchronizeConnectionSender] + background_peer_state: BackgroundPeerState + + def reset_mocks(self): + mocks = ( + self.abort_timeout_sender_mock, + self.synchronize_connection_sender_mock, + self.peer_is_ready_sender_mock, + self.sender_mock, + self.receive_socket_mock, + self.socket_factory_mock, + ) + for mock in mocks: + mock.reset_mock() + + +def create_test_setup() -> TestSetup: + peer = Peer( + connection_info=ConnectionInfo( + name="t1", + ipaddress=IPAddress(ip_address="127.0.0.1"), + port=Port(port=11), + group_identifier="g" + )) + my_connection_info = ConnectionInfo( + name="t0", + ipaddress=IPAddress(ip_address="127.0.0.1"), + port=Port(port=10), + group_identifier="g" + ) + receive_socket_mock = create_autospec(Socket) + socket_factory_mock: Union[MagicMock, SocketFactory] = create_autospec(SocketFactory) + mock_cast(socket_factory_mock.create_socket).side_effect = [receive_socket_mock] + sender_mock = create_autospec(Sender) + abort_timeout_sender_mock = create_autospec(AbortTimeoutSender) + peer_is_ready_sender_mock = create_autospec(PeerIsReadySender) + synchronize_connection_sender_mock = create_autospec(SynchronizeConnectionSender) + background_peer_state = BackgroundPeerState( + my_connection_info=my_connection_info, + peer=peer, + socket_factory=socket_factory_mock, + sender=sender_mock, + abort_timeout_sender=abort_timeout_sender_mock, + peer_is_ready_sender=peer_is_ready_sender_mock, + synchronize_connection_sender=synchronize_connection_sender_mock + ) + return TestSetup( + peer=peer, + my_connection_info=my_connection_info, + socket_factory_mock=socket_factory_mock, + sender_mock=sender_mock, + abort_timeout_sender_mock=abort_timeout_sender_mock, + peer_is_ready_sender_mock=peer_is_ready_sender_mock, + synchronize_connection_sender_mock=synchronize_connection_sender_mock, + background_peer_state=background_peer_state, + receive_socket_mock=receive_socket_mock + ) + + +def test_init(): + test_setup = create_test_setup() + assert ( + test_setup.synchronize_connection_sender_mock.mock_calls == [call.send_if_necessary(force=True)] + and test_setup.peer_is_ready_sender_mock.mock_calls == [] + and test_setup.abort_timeout_sender_mock.mock_calls == [] + and test_setup.sender_mock.mock_calls == [] + and mock_cast(test_setup.socket_factory_mock.create_socket).mock_calls == [call(SocketType.PAIR)] + and test_setup.receive_socket_mock.mock_calls == [ + call.bind('inproc://peer/g/127.0.0.1/11') + ] + ) + + +def test_resend(): + test_setup = create_test_setup() + test_setup.reset_mocks() + test_setup.background_peer_state.resend_if_necessary() + assert ( + test_setup.synchronize_connection_sender_mock.mock_calls == [call.send_if_necessary()] + and test_setup.peer_is_ready_sender_mock.mock_calls == [call.send_if_necessary()] + and test_setup.abort_timeout_sender_mock.mock_calls == [call.send_if_necessary()] + and test_setup.sender_mock.mock_calls == [] + and mock_cast(test_setup.socket_factory_mock.create_socket).mock_calls == [] + and test_setup.receive_socket_mock.mock_calls == [] + ) + + +def test_received_synchronize_connection(): + test_setup = create_test_setup() + test_setup.reset_mocks() + test_setup.background_peer_state.received_synchronize_connection() + assert ( + test_setup.synchronize_connection_sender_mock.mock_calls == [] + and test_setup.peer_is_ready_sender_mock.mock_calls == [call.enable(), call.reset_timer()] + and test_setup.abort_timeout_sender_mock.mock_calls == [call.stop()] + and test_setup.sender_mock.mock_calls == [ + call.send(Message(__root__=AcknowledgeConnectionMessage(source=test_setup.my_connection_info)))] + and mock_cast(test_setup.socket_factory_mock.create_socket).mock_calls == [] + and test_setup.receive_socket_mock.mock_calls == [] + ) + + +def test_received_acknowledge_connection(): + test_setup = create_test_setup() + test_setup.reset_mocks() + test_setup.background_peer_state.received_acknowledge_connection() + assert ( + test_setup.synchronize_connection_sender_mock.mock_calls == [call.stop()] + and test_setup.peer_is_ready_sender_mock.mock_calls == [call.send_if_necessary(force=True)] + and test_setup.abort_timeout_sender_mock.mock_calls == [call.stop()] + and test_setup.sender_mock.mock_calls == [] + and mock_cast(test_setup.socket_factory_mock.create_socket).mock_calls == [] + and test_setup.receive_socket_mock.mock_calls == [] + ) diff --git a/tests/udf_communication/peer_communication/test_peer_is_ready_sender.py b/tests/udf_communication/peer_communication/test_peer_is_ready_sender.py new file mode 100644 index 00000000..5c3412dd --- /dev/null +++ b/tests/udf_communication/peer_communication/test_peer_is_ready_sender.py @@ -0,0 +1,170 @@ +import dataclasses +from typing import Union, cast, Any +from unittest.mock import MagicMock, Mock, create_autospec, call + +from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo +from exasol_advanced_analytics_framework.udf_communication.ip_address import IPAddress, Port +from exasol_advanced_analytics_framework.udf_communication.messages import PeerIsReadyToReceiveMessage +from exasol_advanced_analytics_framework.udf_communication.peer import Peer +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.peer_is_ready_sender import \ + PeerIsReadySender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.timer import Timer +from exasol_advanced_analytics_framework.udf_communication.serialization import serialize_message +from exasol_advanced_analytics_framework.udf_communication.socket_factory.abstract import Socket + + +def mock_cast(obj: Any) -> Mock: + return cast(Mock, obj) + + +@dataclasses.dataclass() +class TestSetup: + peer: Peer + timer_mock: Union[MagicMock, Timer] + out_control_socket_mock: Union[MagicMock, Socket] + peer_is_ready_sender: PeerIsReadySender = None + + def reset_mock(self): + self.out_control_socket_mock.reset_mock() + self.timer_mock.reset_mock() + + +def create_test_setup(): + peer = Peer( + connection_info=ConnectionInfo( + name="t2", + ipaddress=IPAddress(ip_address="127.0.0.1"), + port=Port(port=12), + group_identifier="g" + )) + my_connection_info = ConnectionInfo( + name="t1", + ipaddress=IPAddress(ip_address="127.0.0.1"), + port=Port(port=11), + group_identifier="g" + ) + timer_mock = create_autospec(Timer) + out_control_socket_mock = create_autospec(Socket) + peer_is_ready_sender = PeerIsReadySender( + peer=peer, + my_connection_info=my_connection_info, + out_control_socket=out_control_socket_mock, + timer=timer_mock + ) + return TestSetup( + peer=peer, + timer_mock=timer_mock, + out_control_socket_mock=out_control_socket_mock, + peer_is_ready_sender=peer_is_ready_sender + ) + + +def test_init(): + test_setup = create_test_setup() + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [] + ) + + +def test_send_if_necessary_after_init_and_is_time_false(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = False + test_setup.reset_mock() + + test_setup.peer_is_ready_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [ + call.is_time() + ] + ) + + +def test_send_if_necessary_after_init_and_is_time_false_and_force(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = False + test_setup.reset_mock() + + test_setup.peer_is_ready_sender.send_if_necessary(force=True) + + assert ( + test_setup.out_control_socket_mock.mock_calls == + [ + call.send(serialize_message(PeerIsReadyToReceiveMessage(peer=test_setup.peer))) + ] + and test_setup.timer_mock.mock_calls == [ + call.is_time() + ] + ) + + +def test_send_if_necessary_after_init_and_is_time_true(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.reset_mock() + + test_setup.peer_is_ready_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [ + call.is_time() + ] + ) + + +def test_send_if_necessary_after_enable_and_is_time_false(): + test_setup = create_test_setup() + test_setup.peer_is_ready_sender.enable() + mock_cast(test_setup.timer_mock.is_time).return_value = False + test_setup.reset_mock() + + test_setup.peer_is_ready_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [call.is_time()] + ) + + +def test_send_if_necessary_after_enable_and_is_time_true(): + test_setup = create_test_setup() + test_setup.peer_is_ready_sender.enable() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.reset_mock() + + test_setup.peer_is_ready_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == + [ + call.send(serialize_message(PeerIsReadyToReceiveMessage(peer=test_setup.peer))) + ] + and test_setup.timer_mock.mock_calls == [call.is_time()] + ) + + +def test_send_if_necessary_after_enable_and_is_time_true_twice(): + test_setup = create_test_setup() + test_setup.peer_is_ready_sender.enable() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.peer_is_ready_sender.send_if_necessary() + test_setup.reset_mock() + + test_setup.peer_is_ready_sender.send_if_necessary() + + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [call.is_time()] + ) + + +def test_reset_timer(): + test_setup = create_test_setup() + test_setup.peer_is_ready_sender.reset_timer() + assert ( + test_setup.out_control_socket_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [call.reset_timer()] + ) diff --git a/tests/udf_communication/peer_communication/test_send_recv.py b/tests/udf_communication/peer_communication/test_send_recv.py index 5ccb2748..d2fb6fc5 100644 --- a/tests/udf_communication/peer_communication/test_send_recv.py +++ b/tests/udf_communication/peer_communication/test_send_recv.py @@ -6,11 +6,14 @@ import structlog import zmq from structlog import WriteLoggerFactory +from structlog.tracebacks import ExceptionDictTransformer +from structlog.typing import FilteringBoundLogger from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo from exasol_advanced_analytics_framework.udf_communication.ip_address import IPAddress from exasol_advanced_analytics_framework.udf_communication.peer_communicator import PeerCommunicator from exasol_advanced_analytics_framework.udf_communication.socket_factory.zmq_wrapper import ZMQSocketFactory +from tests.udf_communication.peer_communication.conditional_method_dropper import ConditionalMethodDropper from tests.udf_communication.peer_communication.utils import TestProcess, BidirectionalQueue, assert_processes_finish structlog.configure( @@ -18,14 +21,17 @@ logger_factory=WriteLoggerFactory(file=Path(__file__).with_suffix(".log").open("wt")), processors=[ structlog.contextvars.merge_contextvars, + ConditionalMethodDropper(method_name="debug"), structlog.processors.add_log_level, - structlog.processors.StackInfoRenderer(), - structlog.dev.set_exc_info, structlog.processors.TimeStamper(), + structlog.processors.ExceptionRenderer(exception_formatter=ExceptionDictTransformer(locals_max_string=320)), + structlog.processors.CallsiteParameterAdder(), structlog.processors.JSONRenderer() ] ) +LOGGER: FilteringBoundLogger = structlog.get_logger() + def run(name: str, group_identifier: str, number_of_instances: int, queue: BidirectionalQueue, seed: int = 0): listen_ip = IPAddress(ip_address=f"127.1.0.1") @@ -42,6 +48,7 @@ def run(name: str, group_identifier: str, number_of_instances: int, queue: Bidir for index, connection_infos in peer_connection_infos.items(): com.register_peer(connection_infos) com.wait_for_peers() + LOGGER.info("Peer is ready", name=name) for peer in com.peers(): com.send(peer, [socker_factory.create_frame(name.encode("utf8"))]) received_values: Set[str] = set() @@ -53,17 +60,40 @@ def run(name: str, group_identifier: str, number_of_instances: int, queue: Bidir @pytest.mark.parametrize("number_of_instances, repetitions", [(2, 1000), (10, 100)]) def test_reliability(number_of_instances: int, repetitions: int): - for i in range(repetitions): - group = f"{time.monotonic_ns()}" - expected_received_values, received_values = run_test(group, number_of_instances) - assert expected_received_values == received_values + run_test_with_repetitions(number_of_instances, repetitions) + + +REPETITIONS_FOR_FUNCTIONALITY = 2 + + +def test_functionality_2(): + run_test_with_repetitions(2, REPETITIONS_FOR_FUNCTIONALITY) -def test_functionality(): - group = f"{time.monotonic_ns()}" - number_of_instances = 2 - expected_received_values, received_values = run_test(group, number_of_instances) - assert expected_received_values == received_values +def test_functionality_10(): + run_test_with_repetitions(10, REPETITIONS_FOR_FUNCTIONALITY) + + +def test_functionality_25(): + run_test_with_repetitions(25, REPETITIONS_FOR_FUNCTIONALITY) + + +def run_test_with_repetitions(number_of_instances: int, repetitions: int): + for i in range(repetitions): + LOGGER.info(f"Start iteration", + iteration=i + 1, + repetitions=repetitions, + number_of_instances=number_of_instances) + start_time = time.monotonic() + group = f"{time.monotonic_ns()}" + expected_peers_of_threads, peers_of_threads = run_test(group, number_of_instances) + assert expected_peers_of_threads == peers_of_threads + end_time = time.monotonic() + LOGGER.info(f"Finish iteration", + iteration=i + 1, + repetitions=repetitions, + number_of_instances=number_of_instances, + duration=end_time - start_time) def run_test(group: str, number_of_instances: int): @@ -75,7 +105,7 @@ def run_test(group: str, number_of_instances: int): connection_infos[i] = processes[i].get() for i in range(number_of_instances): t = processes[i].put(connection_infos) - assert_processes_finish(processes, timeout_in_seconds=120) + assert_processes_finish(processes, timeout_in_seconds=180) received_values: Dict[int, Set[str]] = {} for i in range(number_of_instances): received_values[i] = processes[i].get() diff --git a/tests/udf_communication/peer_communication/test_synchronize_connection_sender.py b/tests/udf_communication/peer_communication/test_synchronize_connection_sender.py new file mode 100644 index 00000000..635bfc07 --- /dev/null +++ b/tests/udf_communication/peer_communication/test_synchronize_connection_sender.py @@ -0,0 +1,179 @@ +import dataclasses +from typing import Union +from unittest.mock import MagicMock, create_autospec, call + +from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo +from exasol_advanced_analytics_framework.udf_communication.ip_address import IPAddress, Port +from exasol_advanced_analytics_framework.udf_communication.messages import SynchronizeConnectionMessage, \ + Message +from exasol_advanced_analytics_framework.udf_communication.peer import Peer +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.sender import Sender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.synchronize_connection_sender import \ + SynchronizeConnectionSender +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.timer import Timer +from tests.udf_communication.peer_communication.mock_cast import mock_cast + + +@dataclasses.dataclass() +class TestSetup: + my_connection_info: ConnectionInfo + timer_mock: Union[MagicMock, Timer] + sender_mock: Union[MagicMock, Sender] + synchronize_connection_sender: SynchronizeConnectionSender + + def reset_mocks(self): + self.sender_mock.reset_mock() + self.timer_mock.reset_mock() + + +def create_test_setup(): + peer = Peer( + connection_info=ConnectionInfo( + name="t2", + ipaddress=IPAddress(ip_address="127.0.0.1"), + port=Port(port=12), + group_identifier="g" + )) + my_connection_info = ConnectionInfo( + name="t1", + ipaddress=IPAddress(ip_address="127.0.0.1"), + port=Port(port=11), + group_identifier="g" + ) + timer_mock = create_autospec(Timer) + sender_mock = create_autospec(Sender) + synchronize_connection_sender = SynchronizeConnectionSender( + sender=sender_mock, + timer=timer_mock, + my_connection_info=my_connection_info, + peer=peer + ) + return TestSetup( + sender_mock=sender_mock, + timer_mock=timer_mock, + my_connection_info=my_connection_info, + synchronize_connection_sender=synchronize_connection_sender + ) + + +def test_init(): + test_setup = create_test_setup() + assert ( + test_setup.sender_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [] + ) + + +def test_send_if_necessary_after_init_and_is_time_false(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = False + test_setup.reset_mocks() + + test_setup.synchronize_connection_sender.send_if_necessary() + + assert ( + test_setup.sender_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [call.is_time()] + ) + + +def test_send_if_necessary_after_init_and_is_time_false_and_force(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = False + test_setup.reset_mocks() + + test_setup.synchronize_connection_sender.send_if_necessary(force=True) + + assert ( + test_setup.sender_mock.mock_calls == + [ + call.send(Message(__root__=SynchronizeConnectionMessage(source=test_setup.my_connection_info))) + ] + and test_setup.timer_mock.mock_calls == + [ + call.is_time(), + call.reset_timer() + ] + ) + + +def test_send_if_necessary_after_init_and_is_time_true(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.reset_mocks() + + test_setup.synchronize_connection_sender.send_if_necessary() + + assert ( + test_setup.sender_mock.mock_calls == + [ + call.send(Message(__root__=SynchronizeConnectionMessage(source=test_setup.my_connection_info))) + ] + and test_setup.timer_mock.mock_calls == + [ + call.is_time(), + call.reset_timer() + ] + ) + + +def test_send_if_necessary_twice_and_is_time_true(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.synchronize_connection_sender.send_if_necessary() + test_setup.reset_mocks() + + test_setup.synchronize_connection_sender.send_if_necessary() + + assert ( + test_setup.sender_mock.mock_calls == + [ + call.send(Message(__root__=SynchronizeConnectionMessage(source=test_setup.my_connection_info))) + ] + and test_setup.timer_mock.mock_calls == + [ + call.is_time(), + call.reset_timer() + ] + ) + + +def test_received_acknowledge_connection_after_init(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.reset_mocks() + + test_setup.synchronize_connection_sender.stop() + + assert ( + test_setup.sender_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [] + ) + + +def test_received_acknowledge_connection_after_send(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.synchronize_connection_sender.send_if_necessary() + test_setup.reset_mocks() + + test_setup.synchronize_connection_sender.stop() + + assert ( + test_setup.sender_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [] + ) + + +def test_send_if_necessary_after_received_acknowledge_connection_and_is_time_true(): + test_setup = create_test_setup() + mock_cast(test_setup.timer_mock.is_time).return_value = True + test_setup.synchronize_connection_sender.stop() + test_setup.reset_mocks() + + test_setup.synchronize_connection_sender.send_if_necessary() + + assert ( + test_setup.sender_mock.mock_calls == [] + and test_setup.timer_mock.mock_calls == [call.is_time()] + ) diff --git a/tests/udf_communication/peer_communication/test_timer.py b/tests/udf_communication/peer_communication/test_timer.py new file mode 100644 index 00000000..d8187b44 --- /dev/null +++ b/tests/udf_communication/peer_communication/test_timer.py @@ -0,0 +1,82 @@ +from typing import Union +from unittest.mock import create_autospec, MagicMock, call + +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.clock import Clock +from exasol_advanced_analytics_framework.udf_communication.peer_communicator.timer import Timer +from tests.mock_cast import mock_cast + + +def test_init(): + clock_mock: Union[MagicMock, Clock] = create_autospec(Clock) + + timer = Timer(clock=clock_mock, timeout_in_ms=10) + + assert clock_mock.mock_calls == [call.current_timestamp_in_ms()] + + +def test_is_time_false(): + clock_mock: Union[MagicMock, Clock] = create_autospec(Clock) + mock_cast(clock_mock.current_timestamp_in_ms).side_effect = [0, 10] + timer = Timer(clock=clock_mock, timeout_in_ms=10) + clock_mock.reset_mock() + + result = timer.is_time() + + assert result == False and clock_mock.mock_calls == [call.current_timestamp_in_ms()] + + +def test_is_time_true(): + clock_mock: Union[MagicMock, Clock] = create_autospec(Clock) + mock_cast(clock_mock.current_timestamp_in_ms).side_effect = [0, 11] + timer = Timer(clock=clock_mock, timeout_in_ms=10) + clock_mock.reset_mock() + + result = timer.is_time() + + assert result == True and clock_mock.mock_calls == [call.current_timestamp_in_ms()] + +def test_is_time_true_after_true(): + clock_mock: Union[MagicMock, Clock] = create_autospec(Clock) + mock_cast(clock_mock.current_timestamp_in_ms).side_effect = [0, 11, 12] + timer = Timer(clock=clock_mock, timeout_in_ms=10) + timer.is_time() + clock_mock.reset_mock() + + result = timer.is_time() + + assert result == True and clock_mock.mock_calls == [call.current_timestamp_in_ms()] + + +def test_reset_timer(): + clock_mock: Union[MagicMock, Clock] = create_autospec(Clock) + mock_cast(clock_mock.current_timestamp_in_ms).side_effect = [0, 11] + timer = Timer(clock=clock_mock, timeout_in_ms=10) + clock_mock.reset_mock() + + timer.reset_timer() + + assert clock_mock.mock_calls == [call.current_timestamp_in_ms()] + + +def test_it_time_false_after_reset_timer(): + clock_mock: Union[MagicMock, Clock] = create_autospec(Clock) + mock_cast(clock_mock.current_timestamp_in_ms).side_effect = [0, 10, 20] + timer = Timer(clock=clock_mock, timeout_in_ms=10) + timer.reset_timer() + clock_mock.reset_mock() + + result = timer.is_time() + + assert result == False and clock_mock.mock_calls == [call.current_timestamp_in_ms()] + + +def test_it_time_true_after_reset_timer(): + clock_mock: Union[MagicMock, Clock] = create_autospec(Clock) + mock_cast(clock_mock.current_timestamp_in_ms).side_effect = [0, 10, 21] + timer = Timer(clock=clock_mock, timeout_in_ms=10) + timer.reset_timer() + clock_mock.reset_mock() + + result = timer.is_time() + + assert result == True and clock_mock.mock_calls == [call.current_timestamp_in_ms()] diff --git a/tests/udf_communication/peer_communication/utils.py b/tests/udf_communication/peer_communication/utils.py index 3684d1a9..2979320d 100644 --- a/tests/udf_communication/peer_communication/utils.py +++ b/tests/udf_communication/peer_communication/utils.py @@ -4,8 +4,13 @@ from queue import Queue from typing import Any, Callable, List +import structlog +from structlog.typing import FilteringBoundLogger + NANOSECONDS_PER_SECOND = 10 ** 9 +LOGGER: FilteringBoundLogger = structlog.get_logger(__name__) + class BidirectionalQueue: @@ -57,13 +62,14 @@ def assert_processes_finish(processes: List[TestProcess], timeout_in_seconds: in timeout_in_ns = timeout_in_seconds * NANOSECONDS_PER_SECOND start_time_ns = time.monotonic_ns() while True: - no_alive_processes = not any(get_alive_processes(processes)) + alive_processes = get_alive_processes(processes) + no_alive_processes = not any(alive_processes) if no_alive_processes: break difference_ns = time.monotonic_ns() - start_time_ns if difference_ns > timeout_in_ns: break - time.sleep(0.001) + time.sleep(0.01) alive_processes_before_kill = [process.name for process in get_alive_processes(processes)] kill_alive_processes(processes) if len(get_alive_processes(processes)) > 0: diff --git a/tests/udf_communication/socket_factory/zmq/test_zmq_socket.py b/tests/udf_communication/socket_factory/zmq/test_zmq_socket.py index 8e9c410b..108dfdb0 100644 --- a/tests/udf_communication/socket_factory/zmq/test_zmq_socket.py +++ b/tests/udf_communication/socket_factory/zmq/test_zmq_socket.py @@ -1,4 +1,4 @@ -import time +from typing import Union, Optional from typing import Union, Optional from unittest.mock import create_autospec, MagicMock diff --git a/tests/udf_communication/test_local_discovery.py b/tests/udf_communication/test_local_discovery.py index 00a25244..8969a816 100644 --- a/tests/udf_communication/test_local_discovery.py +++ b/tests/udf_communication/test_local_discovery.py @@ -6,6 +6,7 @@ import structlog import zmq from structlog import WriteLoggerFactory +from structlog.tracebacks import ExceptionDictTransformer from structlog.types import FilteringBoundLogger from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo @@ -16,6 +17,7 @@ from exasol_advanced_analytics_framework.udf_communication.peer_communicator import PeerCommunicator from exasol_advanced_analytics_framework.udf_communication.peer_communicator.peer_communicator import key_for_peer from exasol_advanced_analytics_framework.udf_communication.socket_factory.zmq_wrapper import ZMQSocketFactory +from tests.udf_communication.peer_communication.conditional_method_dropper import ConditionalMethodDropper from tests.udf_communication.peer_communication.utils import TestProcess, BidirectionalQueue, assert_processes_finish structlog.configure( @@ -23,10 +25,11 @@ logger_factory=WriteLoggerFactory(file=Path(__file__).with_suffix(".log").open("wt")), processors=[ structlog.contextvars.merge_contextvars, + ConditionalMethodDropper(method_name="debug"), structlog.processors.add_log_level, - structlog.processors.StackInfoRenderer(), - structlog.dev.set_exc_info, structlog.processors.TimeStamper(), + structlog.processors.ExceptionRenderer(exception_formatter=ExceptionDictTransformer(locals_max_string=320)), + structlog.processors.CallsiteParameterAdder(), structlog.processors.JSONRenderer() ] ) @@ -61,20 +64,40 @@ def run(name: str, group_identifier: str, number_of_instances: int, queue: Bidir @pytest.mark.parametrize("number_of_instances, repetitions", [(2, 1000), (10, 100)]) def test_reliability(number_of_instances: int, repetitions: int): + run_test_with_repetitions(number_of_instances, repetitions) + + +REPETITIONS_FOR_FUNCTIONALITY = 2 + + +def test_functionality_2(): + run_test_with_repetitions(2, REPETITIONS_FOR_FUNCTIONALITY) + + +def test_functionality_10(): + run_test_with_repetitions(10, REPETITIONS_FOR_FUNCTIONALITY) + + +def test_functionality_25(): + run_test_with_repetitions(25, REPETITIONS_FOR_FUNCTIONALITY) + + +def run_test_with_repetitions(number_of_instances: int, repetitions: int): for i in range(repetitions): + LOGGER.info(f"Start iteration", + iteration=i + 1, + repetitions=repetitions, + number_of_instances=number_of_instances) + start_time = time.monotonic() group = f"{time.monotonic_ns()}" expected_peers_of_threads, peers_of_threads = run_test(group, number_of_instances) assert expected_peers_of_threads == peers_of_threads - - -def test_functionality(): - number_of_instances = 2 - group = f"{time.monotonic_ns()}" - logger = LOGGER.bind(group=group, location="test") - logger.info("start") - expected_peers_of_threads, peers_of_threads = run_test(group, number_of_instances) - assert expected_peers_of_threads == peers_of_threads - logger.info("success") + end_time = time.monotonic() + LOGGER.info(f"Finish iteration", + iteration=i + 1, + repetitions=repetitions, + number_of_instances=number_of_instances, + duration=end_time - start_time) def run_test(group: str, number_of_instances: int): @@ -84,7 +107,7 @@ def run_test(group: str, number_of_instances: int): for i in range(number_of_instances): processes[i].start() connection_infos[i] = processes[i].get() - assert_processes_finish(processes, timeout_in_seconds=120) + assert_processes_finish(processes, timeout_in_seconds=180) peers_of_threads: Dict[int, List[ConnectionInfo]] = {} for i in range(number_of_instances): peers_of_threads[i] = processes[i].get() diff --git a/tests/udf_communication/test_messages.py b/tests/udf_communication/test_messages.py index e9dcd769..c5dafe59 100644 --- a/tests/udf_communication/test_messages.py +++ b/tests/udf_communication/test_messages.py @@ -3,7 +3,7 @@ from exasol_advanced_analytics_framework.udf_communication.connection_info import ConnectionInfo from exasol_advanced_analytics_framework.udf_communication.ip_address import Port, IPAddress from exasol_advanced_analytics_framework.udf_communication.messages import RegisterPeerMessage, Message, PingMessage, \ - StopMessage, PayloadMessage, MyConnectionInfoMessage, WeAreReadyToReceiveMessage, AreYouReadyToReceiveMessage, \ + StopMessage, PayloadMessage, MyConnectionInfoMessage, \ PeerIsReadyToReceiveMessage from exasol_advanced_analytics_framework.udf_communication.peer import Peer from exasol_advanced_analytics_framework.udf_communication.serialization import serialize_message, deserialize_message @@ -19,8 +19,6 @@ StopMessage(), PayloadMessage(source=connection_info), MyConnectionInfoMessage(my_connection_info=connection_info), - WeAreReadyToReceiveMessage(source=connection_info), - AreYouReadyToReceiveMessage(source=connection_info), PeerIsReadyToReceiveMessage(peer=peer) ] @@ -35,6 +33,7 @@ def test_message_serialization(message): obj = deserialize_message(byte_string, Message) assert message == obj.__root__ + @pytest.mark.parametrize( "message", [message for message in messages],