diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5ad5a31..d5117cd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,6 +9,7 @@ on: - "mix.exs" - "mix.lock" - "test/**" + - ".github/workflows/ci.yaml" push: branches: - main @@ -16,15 +17,15 @@ on: - "lib/**" - "mix.exs" - "mix.lock" + - ".github/workflows/ci.yaml" jobs: test: runs-on: ubuntu-latest - services: - rabbitmq: - image: rabbitmq:3.11 - ports: - - 5552:5552 + strategy: + fail-fast: false + matrix: + version: ["3_13", "3_12", "3_11"] steps: - uses: erlef/setup-beam@v1 with: @@ -33,10 +34,17 @@ jobs: - uses: actions/checkout@v3 - - name: Enable rabbitmq management plugin - run: | - DOCKER_NAME=$(docker ps --filter ancestor=rabbitmq:3.11 --format "{{.Names}}") - docker exec $DOCKER_NAME rabbitmq-plugins enable rabbitmq_stream + - uses: isbang/compose-action@v1.5.1 + with: + compose-file: "./services/docker-compose.yaml" + services: "rabbitmq_stream_${{ matrix.version }}" + + - name: Wait RabbitMQ is Up + run: sleep 10s + shell: bash + + - name: Create 'invoices' SuperStream + run: docker exec rabbitmq_stream rabbitmq-streams add_super_stream invoices --partitions 3 - name: Install Dependencies run: | @@ -50,4 +58,4 @@ jobs: run: mix compile - name: Run tests - run: mix test + run: mix test --exclude test --include v${{ matrix.version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e68a6a..a6862c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,41 +1,46 @@ # Changelog -## 0.1.0 +## 0.4.0 -Initial release with the following features: +Added support for RabbitMQ 3.13, with Route, Partitions and SuperStreams support. -- Opening connection to RabbitMQ server -- Declaring a Stream -- Creating a Stream Publisher -- Subscribing to Stream Messages -- Initial Hex Release +### 0.4.0 Features -## 0.2.0 +- Support for `:consumerupdate`, `:exchangecommandversions`, `:streamstats`, commands. +- Serialization options for encoding and decoding messages. +- TLS Support +- Functional `single-active-consumer`. +- Initial support for `filter_value` consumer parameter, and `:createsuperstream`, `:deletesuperstream`, `:route`, `:partitions` commands. +- Initial support for SuperStreams, with RabbitMQStream.SuperConsumer and RabbitMQStream.SuperPublisher. -The main objective of this release is to remove the manually added code from `rabbitmq_stream_common`'s Erlang implementation of Encoding and Decoding logic, with frame buffering. +### 0.4.0 Changes -## 0.2.1 +The 'Message' module tree was refactored to make all the Encoding and Decoding logic stay close to each other. -Documentation and Configuration refactoring +- Improved the cleanup logic for closing the connection. +- Publishers and Consumers now expects any name of a GenServer process, instead of a Module. +- Added checks on supported commands based on Server version, and exchanged commands versions. -- It is now possible to define the connection and subscriber parameters throught the `config.exs` file -- Documentation improvements, and examples +### 0.4.0 Breaking Changes + +- Renamed `RabbitMQStream.Subscriber` to `RabbitMQStream.Consumer` +- Renamed `RabbitMQStream.Publisher` to `RabbitMQStream.Producer` ## 0.3.0 -Added an implementation for a stream Subscriber, fixed bugs and improved the documentation. +Added an implementation for a stream Consumer, fixed bugs and improved the documentation. -### Features +### 0.3.0 Features - Added the `:credit` command. - Added `RabbitMQStream.Subscriber`, which subscribes to a stream, while tracking its offset and credit based on customizeable strategies. - Added the possibility of globally configuring the default Connection for Publishers and Subscribers -### Bug Fixes +### 0.3.0 Bug Fixes - Fixed an issue where tcp packages with multiple commands where not being correctly parsed, and in reversed order -### Changes +### 0.3.0 Changes - `RabbitMQStream.Publisher` no longer calls `connect` on the Connection during its setup. - Moved `RabbitMQStream.Publisher`'s setup logic into `handle_continue`, to prevent locking up the application startup. @@ -43,6 +48,27 @@ Added an implementation for a stream Subscriber, fixed bugs and improved the doc - `RabbitMQStream.Publisher` module now can optionally declare a `before_start/2` callback, which is called before it calls `declare_publisher/2`, and can be used to create the stream if it doesn't exists. - `RabbitMQStream.Connection` now buffers the requests while the connection is not yet `:open`. -### Breaking Changes +### 0.3.0 Breaking Changes - Subscription deliver messages are now in the format `{:chunk, %RabbitMQ.OsirisChunk{}}`. + +## 0.2.1 + +Documentation and Configuration refactoring + +- It is now possible to define the connection and subscriber parameters throught the `config.exs` file +- Documentation improvements, and examples + +## 0.2.0 + +The main objective of this release is to remove the manually added code from `rabbitmq_stream_common`'s Erlang implementation of Encoding and Decoding logic, with frame buffering. + +## 0.1.0 + +Initial release with the following features: + +- Opening connection to RabbitMQ server +- Declaring a Stream +- Creating a Stream Publisher +- Subscribing to Stream Messages +- Initial Hex Release diff --git a/README.md b/README.md index d315b28..bd0506c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Zero dependencies Elixir Client for [RabbitMQ Streams Protocol](https://www.rabb ## Usage -### Subscribing to stream +### Consuming from stream First you define a connection @@ -20,23 +20,23 @@ defmodule MyApp.MyConnection do end ``` -You then can declare a subscriber module with the `RabbitMQStream.Subscriber`: +You then can declare a consumer module with the `RabbitMQStream.Consumer`: ```elixir -defmodule MyApp.MySubscriber do - use RabbitMQStream.Subscriber, +defmodule MyApp.MyConsumer do + use RabbitMQStream.Consumer, connection: MyApp.MyConnection, stream_name: "my_stream", initial_offset: :first @impl true - def handle_chunk(%RabbitMQStream.OsirisChunk{}=_chunk, _subscriber) do + def handle_chunk(%RabbitMQStream.OsirisChunk{}=_chunk, _consumer) do :ok end end ``` -Or you could manually subscribe to the stream with +Or you could manually consume from the stream with ```elixir {:ok, _subscription_id} = MyApp.MyConnection.subscribe("stream-01", self(), :next, 999) @@ -51,17 +51,17 @@ def handle_info({:chunk, %RabbitMQStream.OsirisChunk{} = chunk}, state) do end ``` -You can take a look at an example Subscriber GenServer at the [Subscribing Documentation](guides/tutorial/subscribing.md). +You can take a look at an example Consumer GenServer at the [Consuming Documentation](guides/tutorial/consuming.md). ### Publishing to stream -RabbitMQ Streams protocol needs a static `:reference_name` per publisher. This is used to prevent message duplication. For this reason, each stream needs, for now, a static module to publish messages, which keeps track of its own `publishing_id`. +RabbitMQ Streams protocol needs a static `:reference_name` per producer. This is used to prevent message duplication. For this reason, each stream needs, for now, a static module to publish messages, which keeps track of its own `publishing_id`. -You can define a `Publisher` module like this: +You can define a `Producer` module like this: ```elixir -defmodule MyApp.MyPublisher do - use RabbitMQStream.Publisher, +defmodule MyApp.MyProducer do + use RabbitMQStream.Producer, stream: "stream-01", connection: MyApp.MyConnection end @@ -70,7 +70,7 @@ end Then you can publish messages to the stream: ```elixir -MyApp.MyPublisher.publish("Hello World") +MyApp.MyProducer.publish("Hello World") ``` ## Installation @@ -99,4 +99,28 @@ end ``` +You can configure a default Serializer module by passing it to the defaults configuration option + +```elixir +config :rabbitmq_stream, :defaults, + serializer: Jason +end +``` + +## TLS Support + +You can configure the RabbitmqStream to use TLS connections: + +```elixir +coonfig :rabbitmq_stream, :defaults, + connection: [ + transport: :ssl, + ssl_opts: [ + keyfile: "services/cert/client_box_key.pem", + certfile: "services/cert/client_box_certificate.pem", + cacertfile: "services/cert/ca_certificate.pem" + ] + ] +``` + For more information, check the [documentation](https://hexdocs.pm/rabbitmq_stream/). diff --git a/config/test.exs b/config/test.exs index c205b06..7cedea2 100644 --- a/config/test.exs +++ b/config/test.exs @@ -2,3 +2,8 @@ import Config # Prevents the CI from being spammed with logs config :logger, :level, :info + +config :rabbitmq_stream, :defaults, + connection: [ + port: 5553 + ] diff --git a/docs/support-table.md b/docs/support-table.md index aac1f79..9bd3428 100644 --- a/docs/support-table.md +++ b/docs/support-table.md @@ -1,34 +1,34 @@ # Support Table -| Command | Supported? | -|-------------------------|------------| -| declarepublisher | ✔️ | -| publish | ✔️ | -| publishconfirm | ✔️ | -| publisherror | ✔️ | -| querypublishersequence | ✔️ | -| deletepublisher | ✔️ | -| subscribe | ✔️ | -| deliver | ✔️ | -| credit | ✔️ | -| storeoffset | ✔️ | -| queryoffset | ✔️ | -| unsubscribe | ✔️ | -| create | ✔️ | -| delete | ✔️ | -| metadata | ✔️ | -| metadataupdate | ✔️ | -| peerproperties | ✔️ | -| saslhandshake | ✔️ | -| saslauthenticate | ✔️ | -| tune | ✔️ | -| open | ✔️ | -| close | ✔️ | -| heartbeat | ✔️ | -| route | ❌ | -| partitions | ❌ | -| consumerupdate | ❌ | -| exchangecommandversions | ❌ | -| streamstats | ❌ | -| createsuperstream | ❌ | -| deletesuperstream | ❌ | +| Command | Supported? | Minimal version | +|-------------------------|------------|-----------------| +| declarepublisher | ✔️ | 3.9 | +| publish | ✔️ | 3.9 | +| publishconfirm | ✔️ | 3.9 | +| publisherror | ✔️ | 3.9 | +| querypublishersequence | ✔️ | 3.9 | +| deletepublisher | ✔️ | 3.9 | +| subscribe | ✔️ | 3.9 | +| deliver | ✔️ | 3.9 | +| credit | ✔️ | 3.9 | +| storeoffset | ✔️ | 3.9 | +| queryoffset | ✔️ | 3.9 | +| unsubscribe | ✔️ | 3.9 | +| create | ✔️ | 3.9 | +| delete | ✔️ | 3.9 | +| metadata | ✔️ | 3.9 | +| metadataupdate | ✔️ | 3.9 | +| peerproperties | ✔️ | 3.9 | +| saslhandshake | ✔️ | 3.9 | +| saslauthenticate | ✔️ | 3.9 | +| tune | ✔️ | 3.9 | +| open | ✔️ | 3.9 | +| close | ✔️ | 3.9 | +| heartbeat | ✔️ | 3.9 | +| consumerupdate | ✔️ | 3.11 | +| streamstats | ✔️ | 3.11 | +| exchangecommandversions | ✔️ | 3.13 | +| createsuperstream | ✔️ | 3.13 | +| deletesuperstream | ✔️ | 3.13 | +| route | ✔️ | 3.13 | +| partitions | ✔️ | 3.13 | diff --git a/guides/concepts/offset.md b/guides/concepts/offset.md new file mode 100644 index 0000000..7af4d27 --- /dev/null +++ b/guides/concepts/offset.md @@ -0,0 +1,83 @@ +# Offset + +When consuming from a Stream, you might be interested in consuming all the messages all the way from the beggining, the end, or somewhere in between. Wehn we start consuming from a stream, we tell RabbitMQ where we want to start consuming from, which is called the `offset`. + +When starting to consume from a stream, we have some options for the offset: + + + +* `:first` - Consume all the messages from the stream, starting from the beggining. +* `:last` - Consume the last message in the stream at the moment the consumer starts, and all the messages that are published after that. +* `:next` - Consume only the messages that are published after the consumer starts. +* `{:offset, non_neg_integer()}` - Consume all the messages from the stream, starting from the offset provided. Each chunk's offset is present in its metadata. +* `{:timestamp, integer()}` - Consume all the messages from the stream, starting from the message that was published at the timestamp provided. + +## Example + +When calling the `RabbitMQStream.Connection.subscribe/5` callback, we can provide the `:offset` option: + +```elixir +defmodule MyApp.MyConnection + use RabbitMQStream.Connection +end + +{:ok, _} = MyApp.MyConnection.start_link() + + +{:ok, subcription_id} = RabbitMQStream.Connection.subscribe( + "stream-name-01", + self(), + :first, + 50_000, + [] +) +``` + +Or you can provide the `:initial_offset` option to `RabbitMQStream.Consumer`: + +```elixir +defmodule MyApp.MyConsumer do + use RabbitMQStream.Consumer, + connection: MyApp.MyConnection, + stream_name: "my_stream", + offset_reference: "default", # defaults to the module's name. E.g. MyApp.MyConsumer + initial_offset: :first + + @impl true + def handle_chunk(%RabbitMQStream.OsirisChunk{} = _chunk, _consumer) do + :ok + end +end +``` + +## Offset Tracking + +Altough the RabbitMQ server doesn't automatically tracks the offset of each consumer, it provides a `store_offset` command. This allows us to store a piece of information, the `offset`, which is referenced by a `reference_name`, on the stream itself. We can then use it to retreive the offset later on, when starting the Consumer. + +A `RabbitMQStream.Consumer` instance automatically queries the offset under `reference_name` on startup, and uses it as the offset passed to the subscribe command. It automatically stores the offset based on customizeable strategies. + +By default it uses the `RabbitMQStream.Consumer.OffsetTracking.CountStrategy` strategy, storing the offset whenever `count` messages are received. It can be used with: + +```elixir +alias RabbitMQStream.Consumer.OffsetTracking.CountStrategy + +use RabbitMQStream.Consumer, + stream_name: "my_stream", + offset_tracking: [CountStrategy, store_after: 50] + # ... + +# The macro also accepts a simplified alias `count` +use RabbitMQStream.Consumer, + stream_name: "my_stream", + offset_tracking: [count: [store_after: 50]] + +# You can also declare multiple strategies to run concurrently. +use RabbitMQStream.Consumer, + stream_name: "my_stream", + offset_tracking: [ + count: [store_after: 50], + interval: [interval: 10_000] # RabbitMQStream.Consumer.OffsetTracking.IntervalStrategy + ] +``` + +You can also implement you own strategy by implementing the `RabbitMQStream.Consumer.OffsetTracking` behavior, and passing it to the `offset_tracking` option. diff --git a/guides/concepts/producing.md b/guides/concepts/producing.md new file mode 100644 index 0000000..d7c47e3 --- /dev/null +++ b/guides/concepts/producing.md @@ -0,0 +1,5 @@ +# Producing + +## Message De-duplication + +## Filter Value diff --git a/guides/concepts/super-streams.md b/guides/concepts/super-streams.md new file mode 100644 index 0000000..84a63dc --- /dev/null +++ b/guides/concepts/super-streams.md @@ -0,0 +1,9 @@ +# Overview + +## Streams + +## Single Active Consumer + +### Upgrade + +## Super Streams diff --git a/guides/introduction/getting-started.md b/guides/introduction/getting-started.md deleted file mode 100644 index 83b3975..0000000 --- a/guides/introduction/getting-started.md +++ /dev/null @@ -1,59 +0,0 @@ -# Getting Started - -## Installation - -First add RabbitMQ to your `mix.exs` dependencies: - -```elixir -def deps do - [ - {:rabbitmq_stream, "~> 0.3.0"}, - # ... - ] -end -``` - -## Subscribing to stream - -First you define a connection - -```elixir -defmodule MyApp.MyConnection - use RabbitMQStream.Connection -end -``` - -Then you can subscribe to messages from a stream: - -```elixir -{:ok, _subscription_id} = MyApp.MyConnection.subscribe("stream-01", self(), :next, 999) -``` - -The caller process will start receiving messages with the format `{:chunk, %RabbitMQStream.OsirisChunk{}}` - -```elixir -def handle_info({:chunk, %RabbitMQStream.OsirisChunk{} = chunk}, state) do - # do something with message - {:noreply, state} -end -``` - -## Publishing to stream - -RabbitMQ Streams protocol needs a static `:reference_name` per publisher. This is used to prevent message duplication. For this reason, each stream needs, for now, a static module to publish messages, which keeps track of its own `publishing_id`. - -You can define a `Publisher` module like this: - -```elixir -defmodule MyApp.MyPublisher - use RabbitMQStream.Publisher, - stream: "stream-01", - connection: MyApp.MyConnection -end -``` - -Then you can publish messages to the stream: - -```elixir -MyApp.MyPublisher.publish("Hello World") -``` diff --git a/guides/setup/configuration.md b/guides/setup/configuration.md new file mode 100644 index 0000000..f6b4691 --- /dev/null +++ b/guides/setup/configuration.md @@ -0,0 +1,100 @@ +# Configuration + +This library currently implements the following actors: + +- `RabbitMQStream.Connection`: Manages a single TCP/SSL connection to a single RabbitMQ Stream node. +- `RabbitMQStream.Consumer`: Consumes from a single stream, while tracking its offset and credits. Requires an existing `RabbitMQStream.Connection`. +- `RabbitMQStream.Producer`: Manages a single producer to a single stream. Requires an existing `RabbitMQStream.Connection`. + +## Configuration Merging + +We can provide the configuration at many different levels, that will then be merged together. The order of precedence is: + +### 1. Defaults + +At your `config/*.exs` file, you can provide a default configuration that will be passed to all the instances of each of the actors. But some of these options are ignored as they don't really make sense to be set at the default level. For example, the `:stream_name` option is ignored by the `Producer` actor. You can see which options are ignored in the documentation of each actor. + +You can provide the default configuration like this: + +```elixir +config :rabbitmq_stream, :defaults, + connection: [ + vhost: "/", + # RabbitMQStream.Connection.connection_option() + # ... + ], + consumer: [ + connection: MyApp.MyConnection, + # [RabbitMQStream.Consumer.consumer_option()] + # ... + ], + producer: [ + connection: MyApp.MyConnection, + # [RabbitMQStream.Producer.producer_option()] + # ... + ] +``` + +### 2. Module specific configuration + +Each time you implement an instance of the actor, with `use RabbitMQStream.Connection`, for example, you can define its configuration on your config file. + +For example, if we have the following module: + +```elixir +defmodule MyApp.MyConnection + use RabbitMQStream.Connection +end +``` + +We can define its configuration like this: + +```elixir +config :rabbitmq_stream, MyApp.MyConnection, + vhost: "/" +``` + +### 3. When defining the actor + +When defining the actor, you can provide the configuration as an option to the `use` macro: + +```elixir +defmodule MyApp.MyConnection + use RabbitMQStream.Connection, + vhost: "/" +end +``` + +### 4. When starting the actor + +When manually starting the actor, you can provide the configuration as an option to the `start_link` function: + +```elixir +{:ok, _} = MyApp.MyConnection.start_link(vhost: "/") +``` + +Or pass the options when defining it in your supervision tree: + +```elixir +defmodule MyApp.Application do + use Application + + def start(_, _) do + children = [ + {MyApp.MyConnection, vhost: "/"}, + # ... + ] + + opts = # ... + Supervisor.start_link(children, opts) + end +end +``` + +For more information, you can check the documentation at each actor's module; + +## Global Options + +### Serializer + +You can define a Serializer module to be used by the Producer and Consumer modules. It is expected to implement `encode!/1` and `decode!/1` callbacks, and must be defined at compile-time level configurations. diff --git a/guides/setup/getting-started.md b/guides/setup/getting-started.md new file mode 100644 index 0000000..8d096cc --- /dev/null +++ b/guides/setup/getting-started.md @@ -0,0 +1,96 @@ +# Getting Started + +## Installation + +First add RabbitMQ to your `mix.exs` dependencies: + +```elixir +def deps do + [ + {:rabbitmq_stream, "~> 0.3.0"}, + # ... + ] +end +``` + +## Consuming from stream + +First you define a connection + +```elixir +defmodule MyApp.MyConnection + use RabbitMQStream.Connection +end +``` + +You can configure the connection in your `config.exs` file: + +```elixir +config :rabbitmq_stream, MyApp.MyConnection, + vhost: "/" +``` + +Manually starting the connection is as simple as: + +```elixir +{:ok, _} = MyApp.MyConnection.start_link() +``` + +Then you can consume to messages from a stream with: + +```elixir +{:ok, _subscription_id} = MyApp.MyConnection.subscribe("stream-01", self(), :next, 999) +``` + +Which will the start sending the caller process messages containg chunks from the stream, with the format `{:chunk, %RabbitMQStream.OsirisChunk{}}`, which you can handle with, for example, in your `handle_info/2` callback on your GenServer module: + +```elixir +def handle_info({:chunk, %RabbitMQStream.OsirisChunk{} = chunk}, state) do + # do something with message + {:noreply, state} +end +``` + +You can also define a Consumer module that subscribes to a stream, and keeps track of its credits and offset. + +```elixir +defmodule MyApp.MyConsumer do + use RabbitMQStream.Consumer, + connection: MyApp.MyConnection, + stream_name: "my_stream", + initial_offset: :first + + @impl true + def handle_chunk(%RabbitMQStream.OsirisChunk{} = _chunk, _consumer) do + :ok + end +end +``` + +Just add it to your supervision tree, and it will start consuming from the stream. + +```elixir +children = [ + MyApp.MyConnection, + MyApp.MyConsumer + # ... +] +``` + +## Publishing to stream + +To prevent message duplication, RabbitMQ requires us to declare a named Producer before being able to publish messages to a stream. We can do this by `using` the `RabbitMQStream.Producer`, which declare itself to the Connection, with the specified `:reference_name`, defaulting to the module's name. + +```elixir +defmodule MyApp.MyProducer + use RabbitMQStream.Producer, + stream: "my_stream", + connection: MyApp.MyConnection +end +``` + +Then you can publish messages to the stream with: + +```elixir +MyApp.MyProducer.publish("Hello World") +``` diff --git a/guides/tutorial/subscribing.md b/guides/tutorial/subscribing.md deleted file mode 100644 index 5298410..0000000 --- a/guides/tutorial/subscribing.md +++ /dev/null @@ -1,65 +0,0 @@ -# Subscribing to Messages - -After defining a connection with: - -```elixir -defmodule MyApp.MyConnection do - use RabbitMQStream.Connection -end -``` - -You can subscribe to messages from a stream with: - -```elixir -{:ok, _subscription_id} = MyApp.MyConnection.subscribe("stream-01", self(), :next, 999) -``` - -The message will be received with the format `{:chunk, %RabbitMQStream.OsirisChunk{} = chunk}`. - -## Examples - -### Persistent Subscription - -You can `use RabbitMQStream.Subscriber` to create a persistent subscription to a stream, which will automatically track the offset and credit. -You can check more information about the `RabbitMQStream.Subscriber` in its documentation. - -```elixir -defmodule MyApp.MySubscriber do - use RabbitMQStream.Subscriber, - connection: MyApp.MyConnection, - stream_name: "my_stream", - initial_offset: :first - - @impl true - def handle_chunk(_chunk, _subscriber) do - :ok - end -end - -``` - -### Genserver - -An example `GenServer` handler that receives messages from a stream could be written like this: - -```elixir -defmodule MyApp.MySubscriber do - use GenServer - alias RabbitMQStream.OsirisChunk - - def start_link do - GenServer.start_link(__MODULE__, %{}) - end - - def init(state) do - {:ok, subscription_id} = MyApp.MyConnection.subscribe("stream-01", self(), :next, 999) - - {:ok, Map.put(state, :subscription_id, subscription_id)} - end - - def handle_info({:chunk, %OsirisChunk{} = message}, state) do - # do something with message - {:noreply, state} - end -end -``` diff --git a/lib/connection/behavior.ex b/lib/connection/behavior.ex new file mode 100644 index 0000000..8b193bf --- /dev/null +++ b/lib/connection/behavior.ex @@ -0,0 +1,72 @@ +defmodule RabbitMQStream.Connection.Behavior do + @callback start_link([RabbitMQStream.Connection.connection_option() | {:name, atom()}]) :: + :ignore | {:error, any} | {:ok, pid} + + @doc """ + Starts the connection process with the RabbitMQ Stream server, and waits + until the authentication is complete. + + Waits for the connection process if it is already started, and instantly + returns if the connection is already established. + + """ + @callback connect(GenServer.server()) :: :ok | {:error, reason :: atom()} + + @callback close(GenServer.server(), reason :: String.t(), code :: integer()) :: + :ok | {:error, reason :: atom()} + + @callback create_stream(GenServer.server(), String.t(), keyword(String.t())) :: + :ok | {:error, reason :: atom()} + + @callback delete_stream(GenServer.server(), String.t()) :: :ok | {:error, reason :: atom()} + + @callback store_offset(GenServer.server(), String.t(), String.t(), integer()) :: :ok + + @callback query_offset(GenServer.server(), String.t(), String.t()) :: + {:ok, offset :: integer()} | {:error, reason :: atom()} + + @callback declare_producer(GenServer.server(), String.t(), String.t()) :: + {:ok, producer_id :: integer()} | {:error, any()} + + @callback delete_producer(GenServer.server(), producer_id :: integer()) :: + :ok | {:error, reason :: atom()} + + @callback query_metadata(GenServer.server(), [String.t(), ...]) :: + {:ok, metadata :: %{brokers: any(), streams: any()}} + | {:error, reason :: atom()} + + @callback query_producer_sequence(GenServer.server(), String.t(), String.t()) :: + {:ok, sequence :: integer()} | {:error, reason :: atom()} + + @callback publish(GenServer.server(), integer(), integer(), binary()) :: :ok + + @callback subscribe( + GenServer.server(), + stream_name :: String.t(), + pid :: pid(), + offset :: RabbitMQStream.Connection.offset(), + credit :: non_neg_integer(), + properties :: Keyword.t() + ) :: {:ok, subscription_id :: non_neg_integer()} | {:error, reason :: atom()} + + @callback unsubscribe(GenServer.server(), subscription_id :: non_neg_integer()) :: + :ok | {:error, reason :: atom()} + + @doc """ + The server will sometimes send a request to the client, which we must send a response to. + + And example request is the 'ConsumerUpdate', where the server expects a response with the + offset. So the connection sends the request to the subscription handler, which then calls + this function to send the response back to the server. + """ + @callback respond(GenServer.server(), request :: RabbitMQStream.Message.Request.t(), opts :: Keyword.t()) :: :ok + + @doc """ + Adds the specified amount of credits to the subscription under the given `subscription_id`. + + This function instantly returns `:ok` as the RabbitMQ Server only sends a response if the command fails, + which only happens if the subscription is not found. In that case the error is logged. + + """ + @callback credit(GenServer.server(), subscription_id :: non_neg_integer(), credit :: non_neg_integer()) :: :ok +end diff --git a/lib/connection/connection.ex b/lib/connection/connection.ex index 0551e69..488ff0d 100644 --- a/lib/connection/connection.ex +++ b/lib/connection/connection.ex @@ -3,7 +3,7 @@ defmodule RabbitMQStream.Connection do Responsible for encoding and decoding messages, opening and maintaining a socket connection to a single node. It connects to the RabbitMQ, and authenticates, and mantains the connection open with heartbeats. - ## Adding a connectiong to the supervision tree + # Adding a connectiong to the supervision tree You can define a connection with: @@ -25,7 +25,7 @@ defmodule RabbitMQStream.Connection do end - ## Connection configuration + # Connection configuration The connection accept the following options: * `username` - The username to use for authentication. Defaults to `guest`. @@ -38,14 +38,14 @@ defmodule RabbitMQStream.Connection do * `lazy` - If `true`, the connection won't starting until explicitly calling `connect/1`. Defaults to `false`. - ## Subscribing to messages - You can subscribe to messages by calling `subscribe/5`: + # Consuming messages + You can consume messages by calling `subscribe/5`: {:ok, _subscription_id} = MyApp.MyConnection.subscribe("stream-01", self(), :next, 999) - ## Configuration + # Configuration The configuration for the connection can be set in your `config.exs` file: config :rabbitmq_stream, MyApp.MyConnection, @@ -74,15 +74,13 @@ defmodule RabbitMQStream.Connection do The precedence order is is the same order as the examples above, from top to bottom. - ## Buffering + # Buffering Any call or cast to the connection while it is not connected will be buffered and executed once the connection is open. """ defmacro __using__(opts) do - quote bind_quoted: [opts: opts] do - use RabbitMQStream.Connection.Lifecycle - @behaviour RabbitMQStream.Connection + quote bind_quoted: [opts: opts], location: :keep do @opts opts def start_link(opts \\ []) when is_list(opts) do @@ -90,226 +88,309 @@ defmodule RabbitMQStream.Connection do Application.get_env(:rabbitmq_stream, __MODULE__, []) |> Keyword.merge(@opts) |> Keyword.merge(opts) + |> Keyword.put(:name, __MODULE__) - GenServer.start_link(__MODULE__, opts, name: __MODULE__) + RabbitMQStream.Connection.start_link(opts) end - def stop(reason \\ :normal, timeout \\ :infinity) do - GenServer.stop(__MODULE__, reason, timeout) + def child_spec(opts) do + %{id: __MODULE__, start: {__MODULE__, :start_link, [opts]}} end - @impl true - def init(opts) do - conn = %RabbitMQStream.Connection{ - options: [ - host: opts[:host] || "localhost", - port: opts[:port] || 5552, - vhost: opts[:vhost] || "/", - username: opts[:username] || "guest", - password: opts[:password] || "guest", - frame_max: opts[:frame_max] || 1_048_576, - heartbeat: opts[:heartbeat] || 60 - ] - } - - if opts[:lazy] == true do - {:ok, conn} - else - {:ok, conn, {:continue, {:connect}}} - end + def stop(reason \\ :normal, timeout \\ :infinity) do + GenServer.stop(__MODULE__, reason, timeout) end def connect() do - GenServer.call(__MODULE__, {:connect}) + RabbitMQStream.Connection.connect(__MODULE__) end def close(reason \\ "", code \\ 0x00) do - GenServer.call(__MODULE__, {:close, reason, code}) + RabbitMQStream.Connection.close(__MODULE__, reason, code) end - def create_stream(name, arguments \\ []) when is_binary(name) do - GenServer.call(__MODULE__, {:create_stream, arguments ++ [name: name]}) + def create_stream(stream_name, arguments \\ []) do + RabbitMQStream.Connection.create_stream(__MODULE__, stream_name, arguments) end - def delete_stream(name) when is_binary(name) do - GenServer.call(__MODULE__, {:delete_stream, name: name}) + def delete_stream(stream_name) do + RabbitMQStream.Connection.delete_stream(__MODULE__, stream_name) end - def store_offset(stream_name, offset_reference, offset) - when is_binary(stream_name) and - is_binary(offset_reference) and - is_integer(offset) do - GenServer.cast( - __MODULE__, - {:store_offset, stream_name: stream_name, offset_reference: offset_reference, offset: offset} - ) + def store_offset(stream_name, offset_reference, offset) do + RabbitMQStream.Connection.store_offset(__MODULE__, stream_name, offset_reference, offset) end - def query_offset(stream_name, offset_reference) - when is_binary(offset_reference) and - is_binary(stream_name) do - GenServer.call(__MODULE__, {:query_offset, stream_name: stream_name, offset_reference: offset_reference}) + def query_offset(stream_name, offset_reference) do + RabbitMQStream.Connection.query_offset(__MODULE__, stream_name, offset_reference) end - def declare_publisher(stream_name, publisher_reference) - when is_binary(publisher_reference) and - is_binary(stream_name) do - GenServer.call( + def declare_producer(stream_name, producer_reference) do + RabbitMQStream.Connection.declare_producer( __MODULE__, - {:declare_publisher, stream_name: stream_name, publisher_reference: publisher_reference} + stream_name, + producer_reference ) end - def delete_publisher(publisher_id) - when is_integer(publisher_id) and - publisher_id <= 255 do - GenServer.call(__MODULE__, {:delete_publisher, publisher_id: publisher_id}) + def delete_producer(producer_id) do + RabbitMQStream.Connection.delete_producer(__MODULE__, producer_id) end - def query_metadata(streams) - when is_list(streams) and - length(streams) > 0 do - GenServer.call(__MODULE__, {:query_metadata, streams: streams}) + def query_metadata(streams) do + RabbitMQStream.Connection.query_metadata(__MODULE__, streams) end - def query_publisher_sequence(stream_name, publisher_reference) - when is_binary(publisher_reference) and - is_binary(stream_name) do - GenServer.call( + def query_producer_sequence(stream_name, producer_reference) do + RabbitMQStream.Connection.query_producer_sequence( __MODULE__, - {:query_publisher_sequence, publisher_reference: publisher_reference, stream_name: stream_name} + producer_reference, + stream_name ) end - def publish(publisher_id, publishing_id, message) - when is_integer(publisher_id) and - is_binary(message) and - is_integer(publishing_id) and - publisher_id <= 255 do - GenServer.cast( + def publish(producer_id, publishing_id, message, filter_value \\ nil) do + RabbitMQStream.Connection.publish( __MODULE__, - {:publish, publisher_id: publisher_id, published_messages: [{publishing_id, message}], wait: true} + producer_id, + publishing_id, + message, + filter_value ) end - def subscribe(stream_name, pid, offset, credit, properties \\ %{}) - when is_binary(stream_name) and - is_integer(credit) and - is_offset(offset) and - is_map(properties) and - is_pid(pid) and - credit >= 0 do - GenServer.call( + def subscribe(stream_name, pid, offset, credit, properties \\ []) do + RabbitMQStream.Connection.subscribe( __MODULE__, - {:subscribe, stream_name: stream_name, pid: pid, offset: offset, credit: credit, properties: properties} + stream_name, + pid, + offset, + credit, + properties ) end - def unsubscribe(subscription_id) when subscription_id <= 255 do - GenServer.call(__MODULE__, {:unsubscribe, subscription_id: subscription_id}) + def unsubscribe(subscription_id) do + RabbitMQStream.Connection.unsubscribe(__MODULE__, subscription_id) + end + + def credit(subscription_id, credit) do + RabbitMQStream.Connection.credit(__MODULE__, subscription_id, credit) end - def credit(subscription_id, credit) when is_integer(subscription_id) and credit >= 0 do - GenServer.cast(__MODULE__, {:credit, subscription_id: subscription_id, credit: credit}) + def route(routing_key, super_stream) do + RabbitMQStream.Connection.route(__MODULE__, routing_key, super_stream) end - if Mix.env() == :test do - def get_state() do - GenServer.call(__MODULE__, {:get_state}) - end + def stream_stats(stream_name) do + RabbitMQStream.Connection.stream_stats(__MODULE__, stream_name) + end + + def partitions(super_stream) do + RabbitMQStream.Connection.partitions(__MODULE__, super_stream) + end + + def create_super_stream(name, partitions, arguments \\ []) do + RabbitMQStream.Connection.create_super_stream( + __MODULE__, + name, + partitions, + arguments + ) + end + + def delete_super_stream(name) do + RabbitMQStream.Connection.delete_super_stream(__MODULE__, name) + end + + def respond(request, opts) do + RabbitMQStream.Connection.respond(__MODULE__, request, opts) end end end - @type offset :: :first | :last | :next | {:offset, non_neg_integer()} | {:timestamp, integer()} - @type connection_options :: [connection_option] - @type connection_option :: - {:username, String.t()} - | {:password, String.t()} - | {:host, String.t()} - | {:port, non_neg_integer()} - | {:vhost, String.t()} - | {:frame_max, non_neg_integer()} - | {:heartbeat, non_neg_integer()} - | {:lazy, boolean()} + import RabbitMQStream.Connection.Helpers + @behaviour RabbitMQStream.Connection.Behavior - @callback start_link([connection_option | {:name, atom()}]) :: :ignore | {:error, any} | {:ok, pid} + def start_link(opts \\ []) when is_list(opts) do + opts = + Application.get_env(:rabbitmq_stream, :defaults, []) + |> Keyword.get(:connection, []) + |> Keyword.merge(opts) - @callback connect() :: :ok | {:error, reason :: atom()} + GenServer.start_link(RabbitMQStream.Connection.Lifecycle, opts, name: opts[:name]) + end - @callback close(reason :: String.t(), code :: integer()) :: - :ok | {:error, reason :: atom()} + def child_spec(opts) do + %{id: __MODULE__, start: {__MODULE__, :start_link, [opts]}} + end - @callback create_stream(String.t(), keyword(String.t())) :: - :ok | {:error, reason :: atom()} + def connect(server) do + GenServer.call(server, {:connect}) + end - @callback delete_stream(String.t()) :: :ok | {:error, reason :: atom()} + def close(server, reason \\ "", code \\ 0x00) do + GenServer.call(server, {:close, reason, code}) + end - @callback store_offset(String.t(), String.t(), integer()) :: :ok + def create_stream(server, name, arguments \\ []) when is_binary(name) do + GenServer.call(server, {:create_stream, [name: name, arguments: arguments]}) + end - @callback query_offset(String.t(), String.t()) :: - {:ok, offset :: integer()} | {:error, reason :: atom()} + def delete_stream(server, name) when is_binary(name) do + GenServer.call(server, {:delete_stream, name: name}) + end - @callback declare_publisher(String.t(), String.t()) :: - {:ok, publisher_id :: integer()} | {:error, any()} + def store_offset(server, stream_name, offset_reference, offset) + when is_binary(stream_name) and + is_binary(offset_reference) and + is_integer(offset) do + GenServer.cast( + server, + {:store_offset, stream_name: stream_name, offset_reference: offset_reference, offset: offset} + ) + end - @callback delete_publisher(publisher_id :: integer()) :: - :ok | {:error, reason :: atom()} + def query_offset(server, stream_name, offset_reference) + when is_binary(offset_reference) and + is_binary(stream_name) do + GenServer.call(server, {:query_offset, stream_name: stream_name, offset_reference: offset_reference}) + end - @callback query_metadata([String.t(), ...]) :: - {:ok, metadata :: %{brokers: any(), streams: any()}} - | {:error, reason :: atom()} + def declare_producer(server, stream_name, producer_reference) + when is_binary(producer_reference) and + is_binary(stream_name) do + GenServer.call( + server, + {:declare_producer, stream_name: stream_name, producer_reference: producer_reference} + ) + end - @callback query_publisher_sequence(String.t(), String.t()) :: - {:ok, sequence :: integer()} | {:error, reason :: atom()} + def delete_producer(server, producer_id) + when is_integer(producer_id) and + producer_id <= 255 do + GenServer.call(server, {:delete_producer, producer_id: producer_id}) + end - @callback publish(integer(), integer(), binary()) :: :ok + def query_metadata(server, streams) + when is_list(streams) and + length(streams) > 0 do + GenServer.call(server, {:query_metadata, streams: streams}) + end - @callback subscribe( - stream_name :: String.t(), - pid :: pid(), - offset :: offset(), - credit :: non_neg_integer(), - properties :: %{String.t() => String.t()} - ) :: {:ok, subscription_id :: non_neg_integer()} | {:error, reason :: atom()} + def query_producer_sequence(server, stream_name, producer_reference) + when is_binary(producer_reference) and + is_binary(stream_name) do + GenServer.call( + server, + {:query_producer_sequence, producer_reference: producer_reference, stream_name: stream_name} + ) + end - @callback unsubscribe(subscription_id :: non_neg_integer()) :: - :ok | {:error, reason :: atom()} - @doc """ - Adds the specified amount of credits to the subscription under the given `subscription_id`. + def publish(server, producer_id, publishing_id, message, filter_value \\ nil) + when is_integer(producer_id) and + is_binary(message) and + is_integer(publishing_id) and + (filter_value == nil or is_binary(filter_value)) and + producer_id <= 255 do + GenServer.cast( + server, + {:publish, producer_id: producer_id, messages: [{publishing_id, message, filter_value}]} + ) + end - This function instantly returns `:ok` as the RabbitMQ Server only sends a response if the command fails, - which only happens if the subscription is not found. In that case the error is logged. + def subscribe(server, stream_name, pid, offset, credit, properties \\ []) + when is_binary(stream_name) and + is_integer(credit) and + is_offset(offset) and + is_list(properties) and + is_pid(pid) and + credit >= 0 do + GenServer.call( + server, + {:subscribe, stream_name: stream_name, pid: pid, offset: offset, credit: credit, properties: properties} + ) + end - """ - @callback credit(subscription_id :: non_neg_integer(), credit :: non_neg_integer()) :: :ok + def unsubscribe(server, subscription_id) when subscription_id <= 255 do + GenServer.call(server, {:unsubscribe, subscription_id: subscription_id}) + end + + def credit(server, subscription_id, credit) when is_integer(subscription_id) and credit >= 0 do + GenServer.cast(server, {:credit, subscription_id: subscription_id, credit: credit}) + end + + def route(server, routing_key, super_stream) when is_binary(routing_key) and is_binary(super_stream) do + GenServer.call(server, {:route, routing_key: routing_key, super_stream: super_stream}) + end + + def stream_stats(server, stream_name) when is_binary(stream_name) do + GenServer.call(server, {:stream_stats, stream_name: stream_name}) + end + + def partitions(server, super_stream) when is_binary(super_stream) do + GenServer.call(server, {:partitions, super_stream: super_stream}) + end + def create_super_stream(server, name, partitions, arguments \\ []) + when is_binary(name) and + is_list(partitions) and + length(partitions) > 0 do + GenServer.call( + server, + {:create_super_stream, name: name, partitions: partitions, arguments: arguments} + ) + end + + def delete_super_stream(server, name) when is_binary(name) do + GenServer.call(server, {:delete_super_stream, name: name}) + end + + def respond(server, request, opts) when is_list(opts) do + GenServer.cast(server, {:respond, request, opts}) + end + + @type offset :: :first | :last | :next | {:offset, non_neg_integer()} | {:timestamp, integer()} + @type connection_options :: [connection_option] + @type connection_option :: + {:username, String.t()} + | {:password, String.t()} + | {:host, String.t()} + | {:port, non_neg_integer()} + | {:vhost, String.t()} + | {:frame_max, non_neg_integer()} + | {:heartbeat, non_neg_integer()} + | {:lazy, boolean()} @type t() :: %RabbitMQStream.Connection{ options: connection_options, socket: :gen_tcp.socket(), - version: 1, state: :closed | :connecting | :open | :closing, - correlation_sequence: integer(), - publisher_sequence: non_neg_integer(), + correlation_sequence: non_neg_integer(), + producer_sequence: non_neg_integer(), subscriber_sequence: non_neg_integer(), - peer_properties: [[String.t()]], - connection_properties: %{String.t() => String.t() | integer()}, + peer_properties: %{String.t() => term()}, + connection_properties: Keyword.t(), mechanisms: [String.t()], connect_requests: [pid()], request_tracker: %{{atom(), integer()} => {pid(), any()}}, - brokers: %{integer() => BrokerData.t()}, - streams: %{String.t() => StreamData.t()}, subscriptions: %{non_neg_integer() => pid()}, + commands: %{ + RabbitMQStream.Message.Helpers.command() => %{min: non_neg_integer(), max: non_neg_integer()} + }, frames_buffer: RabbitMQStream.Message.Buffer.t(), - request_buffer: :queue.queue({term(), pid()}) + request_buffer: :queue.queue({term(), pid()}), + commands_buffer: :queue.queue({atom(), atom(), list({atom(), term()})}), + # this should not be here. Should find a better way to return the close reason from the 'handler' module + close_reason: String.t() | atom() | nil, + transport: RabbitMQStream.Connection.Transport.t() } - + @enforce_keys [:transport, :options] defstruct [ :socket, + :transport, options: [], - version: 1, correlation_sequence: 1, - publisher_sequence: 1, + producer_sequence: 1, subscriber_sequence: 1, subscriptions: %{}, state: :closed, @@ -318,9 +399,10 @@ defmodule RabbitMQStream.Connection do mechanisms: [], connect_requests: [], request_tracker: %{}, - brokers: %{}, - streams: %{}, + commands: %{}, request_buffer: :queue.new(), - frames_buffer: RabbitMQStream.Message.Buffer.init() + frames_buffer: RabbitMQStream.Message.Buffer.init(), + commands_buffer: :queue.new(), + close_reason: nil ] end diff --git a/lib/connection/handler.ex b/lib/connection/handler.ex index 7057748..f2e02a3 100644 --- a/lib/connection/handler.ex +++ b/lib/connection/handler.ex @@ -2,44 +2,40 @@ defmodule RabbitMQStream.Connection.Handler do @moduledoc false require Logger - alias RabbitMQStream.Message.Encoder - alias RabbitMQStream.Connection - alias RabbitMQStream.Message - alias RabbitMQStream.Message.{Request, Response} + alias RabbitMQStream.Connection + alias RabbitMQStream.Connection.Helpers - def handle_message(%Request{command: :close} = request, conn) do + def handle_message(%Connection{} = conn, %Request{command: :close} = request) do Logger.debug("Connection close requested by server: #{request.data.code} #{request.data.reason}") Logger.debug("Connection closed") - %{conn | state: :closing} - |> send_response(:close, correlation_id: request.correlation_id, code: :ok) - |> handle_closed(request.data.reason) + %{conn | state: :closing, close_reason: request.data.reason} + |> Helpers.push(:response, :close, correlation_id: request.correlation_id, code: :ok) end - def handle_message(%Request{command: :tune} = request, conn) do + def handle_message(%Connection{} = conn, %Request{command: :tune} = request) do Logger.debug("Tunning complete. Starting heartbeat timer.") Process.send_after(self(), {:heartbeat}, conn.options[:heartbeat] * 1000) options = Keyword.merge(conn.options, frame_max: request.data.frame_max, heartbeat: request.data.heartbeat) - %{conn | options: options} - |> send_response(:tune, correlation_id: 0) - |> Map.put(:state, :opening) - |> send_request(:open) + %{conn | options: options, state: :opening} + |> Helpers.push(:response, :tune, correlation_id: 0) + |> Helpers.push(:request, :open) end - def handle_message(%Request{command: :heartbeat}, conn) do + def handle_message(%Connection{} = conn, %Request{command: :heartbeat}) do conn end - def handle_message(%Request{command: :metadata_update} = request, conn) do + def handle_message(%Connection{} = conn, %Request{command: :metadata_update} = request) do conn - |> send_request(:query_metadata, streams: [request.data.stream_name]) + |> Helpers.push(:request, :query_metadata, streams: [request.data.stream_name]) end - def handle_message(%Request{command: :deliver} = response, conn) do + def handle_message(%Connection{} = conn, %Request{command: :deliver} = response) do pid = Map.get(conn.subscriptions, response.data.subscription_id) if pid != nil do @@ -49,24 +45,22 @@ defmodule RabbitMQStream.Connection.Handler do conn end - def handle_message(%Request{command: command}, conn) + def handle_message(%Connection{} = conn, %Request{command: command}) when command in [:publish_confirm, :publish_error] do conn end - def handle_message(%Response{command: :close} = response, conn) do + def handle_message(%Connection{} = conn, %Response{command: :close} = response) do Logger.debug("Connection closed: #{conn.options[:host]}:#{conn.options[:port]}") - {{pid, _data}, conn} = pop_request_tracker(conn, :close, response.correlation_id) - - conn = %{conn | state: :closed, socket: nil} + {{pid, _data}, conn} = Helpers.pop_tracker(conn, :close, response.correlation_id) GenServer.reply(pid, :ok) - conn + %{conn | state: :closing} end - def handle_message(%Response{code: code}, conn) + def handle_message(%Connection{} = conn, %Response{code: code}) when code in [ :sasl_mechanism_not_supported, :authentication_failure, @@ -81,28 +75,33 @@ defmodule RabbitMQStream.Connection.Handler do GenServer.reply(request, {:error, code}) end - %{conn | state: :closed, socket: nil, connect_requests: []} + %{conn | state: :closing, close_reason: code} end - def handle_message(%Response{command: :credit, code: code}, conn) + def handle_message(%Connection{} = conn, %Response{command: :credit, code: code}) when code not in [:ok, nil] do Logger.error("Failed to credit subscription. Reason: #{code}") conn end - def handle_message(%Response{command: command, code: code} = response, conn) + def handle_message(%Connection{} = conn, %Response{command: command, code: code} = response) when command in [ :create_stream, :delete_stream, :query_offset, - :declare_publisher, - :delete_publisher, + :declare_producer, + :delete_producer, :subscribe, - :unsubscribe + :unsubscribe, + :stream_stats, + :create_super_stream, + :delete_super_stream, + :route, + :partitions ] and code not in [:ok, nil] do - {{pid, _data}, conn} = pop_request_tracker(conn, command, response.correlation_id) + {{pid, _data}, conn} = Helpers.pop_tracker(conn, command, response.correlation_id) if pid != nil do GenServer.reply(pid, {:error, code}) @@ -111,43 +110,51 @@ defmodule RabbitMQStream.Connection.Handler do conn end - def handle_message(_, %Connection{state: :closed} = conn) do + def handle_message(%Connection{state: :closed} = conn, _) do Logger.error("Message received on a closed connection") conn end - def handle_message(%Response{command: :peer_properties} = response, conn) do - Logger.debug("Exchange successful.") - Logger.debug("Initiating SASL handshake.") + def handle_message(%Connection{} = conn, %Response{command: :peer_properties} = response) do + Logger.debug("Peer Properties exchange successful. Initiating SASL handshake.") + + # We need to extract the base version from the version string so we can compare + # make decisions based on the version of the server. + version = + ~r/(\d+)\.(\d+)\.(\d+)/ + |> Regex.run(response.data.peer_properties["version"], capture: :all_but_first) + |> Enum.map(&String.to_integer/1) + + peer_properties = Map.put(response.data.peer_properties, "base-version", version) - %{conn | peer_properties: response.data.peer_properties} - |> send_request(:sasl_handshake) + %{conn | peer_properties: peer_properties} + |> Helpers.push(:request, :sasl_handshake) end - def handle_message(%Response{command: :sasl_handshake} = response, conn) do + def handle_message(%Connection{} = conn, %Response{command: :sasl_handshake} = response) do Logger.debug("SASL handshake successful. Initiating authentication.") %{conn | mechanisms: response.data.mechanisms} - |> send_request(:sasl_authenticate) + |> Helpers.push(:request, :sasl_authenticate) end - def handle_message(%Response{command: :sasl_authenticate, data: %{sasl_opaque_data: ""}}, conn) do + def handle_message(%Connection{} = conn, %Response{command: :sasl_authenticate, data: %{sasl_opaque_data: ""}}) do Logger.debug("Authentication successful. Initiating connection tuning.") conn end - def handle_message(%Response{command: :sasl_authenticate}, conn) do + def handle_message(%Connection{} = conn, %Response{command: :sasl_authenticate}) do Logger.debug("Authentication successful. Skipping connection tuning.") Logger.debug("Opening connection to vhost: \"#{conn.options[:vhost]}\"") conn - |> send_request(:open) + |> Helpers.push(:request, :open) |> Map.put(:state, :opening) end - def handle_message(%Response{command: :tune} = response, conn) do + def handle_message(%Connection{} = conn, %Response{command: :tune} = response) do Logger.debug("Tunning data received. Starting heartbeat timer.") Logger.debug("Opening connection to vhost: \"#{conn.options[:vhost]}\"") @@ -155,10 +162,15 @@ defmodule RabbitMQStream.Connection.Handler do %{conn | options: options} |> Map.put(:state, :opening) - |> send_request(:open) + |> Helpers.push(:request, :open) end - def handle_message(%Response{command: :open} = response, conn) do + # If the server has a version lower than 3.13, this is the 'terminating' response. + def handle_message( + %Connection{peer_properties: %{"base-version" => version}} = conn, + %Response{command: :open} = response + ) + when version < [3, 13] do Logger.debug("Successfully opened connection with vhost: \"#{conn.options[:vhost]}\"") for request <- conn.connect_requests do @@ -170,30 +182,31 @@ defmodule RabbitMQStream.Connection.Handler do %{conn | state: :open, connect_requests: [], connection_properties: response.data.connection_properties} end - def handle_message({:response, correlation_id, {:metadata, brokers, streams}}, conn) do - {{pid, _data}, conn} = pop_request_tracker(conn, :query_metadata, correlation_id) - - brokers = Map.new(brokers) - - if pid != nil do - GenServer.reply(pid, {:ok, %{brokers: brokers, streams: streams}}) - end - - %{conn | brokers: Map.merge(conn.brokers, brokers), streams: Map.merge(conn.streams, streams)} + def handle_message( + %Connection{peer_properties: %{"base-version" => version}} = conn, + %Response{command: :open} = response + ) + when version >= [3, 13] do + Logger.debug( + "Successfully opened connection with vhost: \"#{conn.options[:vhost]}\". Initiating command version exchange." + ) + + %{conn | connection_properties: response.data.connection_properties} + |> Helpers.push(:request, :exchange_command_versions) end - def handle_message(%Response{command: :query_metadata} = response, conn) do - {{pid, _data}, conn} = pop_request_tracker(conn, :query_metadata, response.correlation_id) + def handle_message(%Connection{} = conn, %Response{command: :query_metadata} = response) do + {{pid, _data}, conn} = Helpers.pop_tracker(conn, :query_metadata, response.correlation_id) if pid != nil do GenServer.reply(pid, {:ok, response.data}) end - %{conn | streams: response.data.streams, brokers: response.data.brokers} + conn end - def handle_message(%Response{command: :query_offset} = response, conn) do - {{pid, _data}, conn} = pop_request_tracker(conn, :query_offset, response.correlation_id) + def handle_message(%Connection{} = conn, %Response{command: :query_offset} = response) do + {{pid, _data}, conn} = Helpers.pop_tracker(conn, :query_offset, response.correlation_id) if pid != nil do GenServer.reply(pid, {:ok, response.data.offset}) @@ -202,8 +215,8 @@ defmodule RabbitMQStream.Connection.Handler do conn end - def handle_message(%Response{command: :declare_publisher} = response, conn) do - {{pid, id}, conn} = pop_request_tracker(conn, :declare_publisher, response.correlation_id) + def handle_message(%Connection{} = conn, %Response{command: :declare_producer} = response) do + {{pid, id}, conn} = Helpers.pop_tracker(conn, :declare_producer, response.correlation_id) if pid != nil do GenServer.reply(pid, {:ok, id}) @@ -212,8 +225,8 @@ defmodule RabbitMQStream.Connection.Handler do conn end - def handle_message(%Response{command: :query_publisher_sequence} = response, conn) do - {{pid, _data}, conn} = pop_request_tracker(conn, :query_publisher_sequence, response.correlation_id) + def handle_message(%Connection{} = conn, %Response{command: :query_producer_sequence} = response) do + {{pid, _data}, conn} = Helpers.pop_tracker(conn, :query_producer_sequence, response.correlation_id) if pid != nil do GenServer.reply(pid, {:ok, response.data.sequence}) @@ -222,20 +235,20 @@ defmodule RabbitMQStream.Connection.Handler do conn end - def handle_message(%Response{command: :subscribe} = response, conn) do - {{pid, data}, conn} = pop_request_tracker(conn, :subscribe, response.correlation_id) + def handle_message(%Connection{} = conn, %Response{command: :subscribe} = response) do + {{pid, data}, conn} = Helpers.pop_tracker(conn, :subscribe, response.correlation_id) - {subscription_id, subscriber} = data + {subscription_id, consumer} = data if pid != nil do GenServer.reply(pid, {:ok, subscription_id}) end - %{conn | subscriptions: Map.put(conn.subscriptions, subscription_id, subscriber)} + %{conn | subscriptions: Map.put(conn.subscriptions, subscription_id, consumer)} end - def handle_message(%Response{command: :unsubscribe} = response, conn) do - {{pid, subscription_id}, conn} = pop_request_tracker(conn, :unsubscribe, response.correlation_id) + def handle_message(%Connection{} = conn, %Response{command: :unsubscribe} = response) do + {{pid, subscription_id}, conn} = Helpers.pop_tracker(conn, :unsubscribe, response.correlation_id) if pid != nil do GenServer.reply(pid, :ok) @@ -244,69 +257,60 @@ defmodule RabbitMQStream.Connection.Handler do %{conn | subscriptions: Map.drop(conn.subscriptions, [subscription_id])} end - def handle_message(%Response{command: command} = response, conn) - when command in [:create_stream, :delete_stream, :delete_publisher] do - {{pid, _data}, conn} = pop_request_tracker(conn, command, response.correlation_id) + # If the server has a version 3.12 or higher, this is the 'terminating' response. + def handle_message(%Connection{} = conn, %Response{command: :exchange_command_versions} = response) do + {{pid, _}, conn} = Helpers.pop_tracker(conn, :exchange_command_versions, response.correlation_id) if pid != nil do - GenServer.reply(pid, :ok) + GenServer.reply(pid, {:ok, response.data}) end - conn - end + commands = + Map.new(response.data.commands, fn command -> + {command.key, %{min: command.min_version, max: command.max_version}} + end) - def push_request_tracker(%Connection{} = conn, type, from, data \\ nil) when is_atom(type) when is_pid(from) do - request_tracker = Map.put(conn.request_tracker, {type, conn.correlation_sequence}, {from, data}) + for request <- conn.connect_requests do + GenServer.reply(request, :ok) + end - %{conn | request_tracker: request_tracker} + send(self(), :flush_request_buffer) + %{conn | state: :open, connect_requests: [], commands: commands} end - def pop_request_tracker(%Connection{} = conn, type, correlation) when is_atom(type) do - {entry, request_tracker} = Map.pop(conn.request_tracker, {type, correlation}, {nil, nil}) + def handle_message(%Connection{} = conn, %Response{command: command, data: data} = response) + when command in [:route, :partitions, :stream_stats] do + {{pid, _data}, conn} = Helpers.pop_tracker(conn, command, response.correlation_id) - {entry, %{conn | request_tracker: request_tracker}} + if pid != nil do + GenServer.reply(pid, {:ok, data}) + end + + conn end - def handle_closed(%Connection{} = conn, reason) do - for request <- conn.connect_requests do - GenServer.reply(request, {:error, :closed}) - end + def handle_message(%Connection{} = conn, %Response{command: command} = response) + when command in [:create_stream, :delete_stream, :delete_producer, :create_super_stream, :delete_super_stream] do + {{pid, _data}, conn} = Helpers.pop_tracker(conn, command, response.correlation_id) - for {client, _data} <- Map.values(conn.request_tracker) do - GenServer.reply(client, {:error, reason}) + if pid != nil do + GenServer.reply(pid, :ok) end - %{conn | request_tracker: %{}, connect_requests: []} + conn end - def send_request(%Connection{} = conn, command, opts \\ []) do - {correlation_sum, opts} = Keyword.pop(opts, :correlation_sum, 1) - {publisher_sum, opts} = Keyword.pop(opts, :publisher_sum, 0) - {subscriber_sum, opts} = Keyword.pop(opts, :subscriber_sum, 0) + # We forward the request because the consumer is the one responsible for + # deciding how to respond to the request. + def handle_message(%Connection{} = conn, %Request{command: :consumer_update} = request) do + subscription_pid = Map.get(conn.subscriptions, request.data.subscription_id) - frame = + if subscription_pid != nil do + send(subscription_pid, {:command, request}) conn - |> Message.Request.new!(command, opts) - |> Encoder.encode!() - - :ok = :gen_tcp.send(conn.socket, frame) - - correlation_sequence = conn.correlation_sequence + correlation_sum - publisher_sequence = conn.publisher_sequence + publisher_sum - subscriber_sequence = conn.subscriber_sequence + subscriber_sum - - %{ + else conn - | correlation_sequence: correlation_sequence, - publisher_sequence: publisher_sequence, - subscriber_sequence: subscriber_sequence - } - end - - def send_response(%Connection{} = conn, command, opts) do - frame = Message.Response.new!(conn, command, opts) |> Encoder.encode!() - :ok = :gen_tcp.send(conn.socket, frame) - - conn + |> Helpers.push(:response, :consumer_update, correlation_id: request.correlation_id, code: :internal_error) + end end end diff --git a/lib/connection/helpers.ex b/lib/connection/helpers.ex new file mode 100644 index 0000000..fc5b833 --- /dev/null +++ b/lib/connection/helpers.ex @@ -0,0 +1,23 @@ +defmodule RabbitMQStream.Connection.Helpers do + def push_tracker(conn, type, from, data \\ nil) when is_atom(type) when is_pid(from) do + request_tracker = Map.put(conn.request_tracker, {type, conn.correlation_sequence}, {from, data}) + + %{conn | request_tracker: request_tracker} + end + + def pop_tracker(conn, type, correlation) when is_atom(type) do + {entry, request_tracker} = Map.pop(conn.request_tracker, {type, correlation}, {nil, nil}) + + {entry, %{conn | request_tracker: request_tracker}} + end + + def push(conn, action, command, opts \\ []) do + commands_buffer = :queue.in({action, command, opts}, conn.commands_buffer) + + %{conn | commands_buffer: commands_buffer} + end + + defguard is_offset(offset) + when offset in [:first, :last, :next] or + (is_tuple(offset) and tuple_size(offset) == 2 and elem(offset, 0) in [:offset, :timestamp]) +end diff --git a/lib/connection/lifecycle.ex b/lib/connection/lifecycle.ex index 992d645..c1024cf 100644 --- a/lib/connection/lifecycle.ex +++ b/lib/connection/lifecycle.ex @@ -1,251 +1,370 @@ defmodule RabbitMQStream.Connection.Lifecycle do @moduledoc false - defmacro __using__(_) do - quote location: :keep do - require Logger - use GenServer - - alias RabbitMQStream.Connection.Handler - alias RabbitMQStream.Connection - alias RabbitMQStream.Message.Buffer - - @impl true - def handle_call({:get_state}, _from, %Connection{} = conn) do - {:reply, conn, conn} + require Logger + + use GenServer + + alias RabbitMQStream.Message.Request + alias RabbitMQStream.Connection + alias RabbitMQStream.Connection.{Handler, Helpers} + + alias RabbitMQStream.Message + alias RabbitMQStream.Message.{Buffer, Encoder} + + @impl GenServer + def init(opts) do + {transport, opts} = Keyword.pop(opts, :transport, :tcp) + + opts = + opts + |> Keyword.put_new(:host, "localhost") + |> Keyword.put_new(:port, 5552) + |> Keyword.put_new(:vhost, "/") + |> Keyword.put_new(:username, "guest") + |> Keyword.put_new(:password, "guest") + |> Keyword.put_new(:frame_max, 1_048_576) + |> Keyword.put_new(:heartbeat, 60) + |> Keyword.put_new(:transport, :tcp) + + transport = + case transport do + :tcp -> RabbitMQStream.Connection.Transport.TCP + :ssl -> RabbitMQStream.Connection.Transport.SSL + transport -> transport end - def handle_call({:connect}, from, %Connection{state: :closed} = conn) do - Logger.debug("Connecting to server: #{conn.options[:host]}:#{conn.options[:port]}") + conn = %RabbitMQStream.Connection{options: opts, transport: transport} - with {:ok, socket} <- - :gen_tcp.connect(String.to_charlist(conn.options[:host]), conn.options[:port], [:binary, active: true]), - :ok <- :gen_tcp.controlling_process(socket, self()) do - Logger.debug("Connection stablished. Initiating properties exchange.") + if opts[:lazy] == true do + {:ok, conn} + else + {:ok, conn, {:continue, {:connect}}} + end + end - conn = - %{conn | socket: socket, state: :connecting, connect_requests: [from]} - |> Handler.send_request(:peer_properties) + @impl GenServer + def handle_call({:connect}, from, %Connection{state: :closed} = conn) do + Logger.debug("Connecting to server: #{conn.options[:host]}:#{conn.options[:port]}") - {:noreply, conn} - else - err -> - Logger.error("Failed to connect to #{conn.options[:host]}:#{conn.options[:port]}") - {:reply, {:error, err}, conn} - end - end + with {:ok, conn} <- connect(conn) do + Logger.debug("Connection stablished. Initiating properties exchange.") - def handle_call({:connect}, _from, %Connection{state: :open} = conn) do - {:reply, :ok, conn} - end + conn = + %{conn | connect_requests: [from | conn.connect_requests]} + |> send_request(:peer_properties) - def handle_call({:connect}, from, %Connection{} = conn) do - conn = %{conn | connect_requests: conn.connect_requests ++ [from]} - {:noreply, conn} - end + {:noreply, conn} + else + err -> + Logger.error("Failed to connect to #{conn.options[:host]}:#{conn.options[:port]}") + {:reply, {:error, err}, conn} + end + end - # Replies with `:ok` if the connection is already closed. Not sure if this behavior is the best. - def handle_call({:close, _reason, _code}, _from, %Connection{state: :closed} = conn) do - {:reply, :ok, conn} - end + def handle_call({:connect}, _from, %Connection{state: :open} = conn) do + {:reply, :ok, conn} + end - def handle_call(action, from, %Connection{state: state} = conn) when state != :open do - {:noreply, %{conn | request_buffer: :queue.in({:call, {action, from}}, conn.request_buffer)}} - end + def handle_call({:connect}, from, %Connection{} = conn) do + {:noreply, %{conn | connect_requests: [from | conn.connect_requests]}} + end - def handle_call({:close, reason, code}, from, %Connection{} = conn) do - Logger.debug("Connection close requested by client: #{reason} #{code}") + # Replies with `:ok` if the connection is already closed. Not sure if this behavior is the best. + def handle_call({:close, _reason, _code}, _from, %Connection{state: :closed} = conn) do + {:reply, :ok, conn} + end - conn = - %{conn | state: :closing} - |> Handler.push_request_tracker(:close, from) - |> Handler.send_request(:close, reason: reason, code: code) + def handle_call(action, from, %Connection{state: state} = conn) when state != :open do + {:noreply, %{conn | request_buffer: :queue.in({:call, {action, from}}, conn.request_buffer)}} + end - {:noreply, conn} - end + def handle_call({:close, reason, code}, from, %Connection{} = conn) do + Logger.debug("Connection close requested by client: #{reason} #{code}") - def handle_call({:subscribe, opts}, from, %Connection{} = conn) do - subscription_id = conn.subscriber_sequence + conn = + conn + |> Helpers.push_tracker(:close, from) + |> send_request(:close, reason: reason, code: code) - conn = - conn - |> Handler.push_request_tracker(:subscribe, from, {subscription_id, opts[:pid]}) - |> Handler.send_request(:subscribe, opts ++ [subscriber_sum: 1, subscription_id: subscription_id]) + {:noreply, conn} + end - {:noreply, conn} - end + def handle_call({:subscribe, opts}, from, %Connection{} = conn) do + {id, conn} = Map.get_and_update!(conn, :subscriber_sequence, &{&1, &1 + 1}) - def handle_call({:unsubscribe, opts}, from, %Connection{} = conn) do - conn = - conn - |> Handler.push_request_tracker(:unsubscribe, from, opts[:subscription_id]) - |> Handler.send_request(:unsubscribe, opts) + conn = + conn + |> Helpers.push_tracker(:subscribe, from, {id, opts[:pid]}) + |> send_request(:subscribe, opts ++ [subscription_id: id]) - {:noreply, conn} - end + {:noreply, conn} + end - def handle_call({command, opts}, from, %Connection{} = conn) - when command in [ - :query_offset, - :delete_publisher, - :query_metadata, - :query_publisher_sequence, - :delete_stream, - :create_stream - ] do - conn = - conn - |> Handler.push_request_tracker(command, from) - |> Handler.send_request(command, opts) + def handle_call({:unsubscribe, opts}, from, %Connection{} = conn) do + conn = + conn + |> Helpers.push_tracker(:unsubscribe, from, opts[:subscription_id]) + |> send_request(:unsubscribe, opts) - {:noreply, conn} - end + {:noreply, conn} + end - def handle_cast({:store_offset, opts}, %Connection{} = conn) do - conn = - conn - |> Handler.send_request(:store_offset, opts) + def handle_call({command, opts}, from, %Connection{} = conn) + when command in [ + :query_offset, + :delete_producer, + :query_metadata, + :query_producer_sequence, + :delete_stream, + :create_stream, + :stream_stats + ] do + conn = + conn + |> Helpers.push_tracker(command, from) + |> send_request(command, opts) + + {:noreply, conn} + end - {:noreply, conn} - end + def handle_call({command, opts}, from, %Connection{} = conn) + when command in [:route, :partitions, :create_super_stream, :delete_super_stream] and + is_map_key(conn.commands, command) do + conn = + conn + |> Helpers.push_tracker(command, from) + |> send_request(command, opts) - def handle_call({:declare_publisher, opts}, from, %Connection{} = conn) do - conn = - conn - |> Handler.push_request_tracker(:declare_publisher, from, conn.publisher_sequence) - |> Handler.send_request(:declare_publisher, opts ++ [publisher_sum: 1]) + {:noreply, conn} + end - {:noreply, conn} - end + def handle_call({command, _opts}, _from, %Connection{peer_properties: %{"version" => version}} = conn) + when command in [:route, :partitions, :create_super_stream, :delete_super_stream] do + Logger.error("Command #{command} is not supported by the server. Its current informed version is '#{version}'.") - @impl true - def handle_cast(action, %Connection{state: state} = conn) when state != :open do - {:noreply, %{conn | request_buffer: :queue.in({:cast, action}, conn.request_buffer)}} - end + {:reply, {:error, :unsupported}, conn} + end - def handle_cast({:publish, opts}, %Connection{} = conn) do - {_wait, opts} = Keyword.pop(opts, :wait, false) + def handle_call({:declare_producer, opts}, from, %Connection{} = conn) do + {id, conn} = Map.get_and_update!(conn, :producer_sequence, &{&1, &1 + 1}) - # conn = - # if wait do - # publishing_ids = Enum.map(opts[:published_messages], fn {id, _} -> id end) - # publish_tracker = PublishingTracker.push(conn.publish_tracker, opts[:publisher_id], publishing_ids, from) - # %{conn | publish_tracker: publish_tracker} - # else - # conn - # end + conn = + conn + |> Helpers.push_tracker(:declare_producer, from, id) + |> send_request(:declare_producer, opts ++ [id: id]) - conn = - conn - |> Handler.send_request(:publish, opts ++ [correlation_sum: 0]) + {:noreply, conn} + end - {:noreply, conn} - end + @impl GenServer + def handle_cast(action, %Connection{state: state} = conn) when state != :open do + {:noreply, %{conn | request_buffer: :queue.in({:cast, action}, conn.request_buffer)}} + end + + def handle_cast({:store_offset, opts}, %Connection{} = conn) do + conn = + conn + |> send_request(:store_offset, opts) + + {:noreply, conn} + end + + def handle_cast({:publish, opts}, %Connection{} = conn) do + conn = + case {opts[:message], conn} do + {{_, _, filter_value}, conn} when is_binary(filter_value) and conn.commands.publish.max >= 2 -> + Logger.error("Publishing a message with a `filter_value` is only supported by RabbitMQ on versions >= 3.13") - def handle_cast({:credit, opts}, %Connection{} = conn) do - conn = conn - |> Handler.send_request(:credit, opts) - {:noreply, conn} + _ -> + conn + |> send_request(:publish, opts ++ [correlation_sum: 0]) end - @impl true - def handle_info({:tcp, _socket, data}, conn) do - {commands, frames_buffer} = - data - |> Buffer.incoming_data(conn.frames_buffer) - |> Buffer.all_commands() + {:noreply, conn} + end - conn = %{conn | frames_buffer: frames_buffer} + def handle_cast({:credit, opts}, %Connection{} = conn) do + conn = + conn + |> send_request(:credit, opts) - conn = Enum.reduce(commands, conn, &Handler.handle_message/2) + {:noreply, conn} + end - cond do - conn.state == :closed -> - {:noreply, conn, :hibernate} + def handle_cast({:respond, %Request{} = request, opts}, %Connection{} = conn) do + conn = + conn + |> send_response(request.command, [correlation_id: request.correlation_id] ++ opts) - true -> - {:noreply, conn} - end - end + {:noreply, conn} + end + + @impl GenServer + def handle_info({key, _socket, data}, conn) when key in [:ssl, :tcp] do + {commands, frames_buffer} = + data + |> Buffer.incoming_data(conn.frames_buffer) + |> Buffer.all_commands() + + conn = %{conn | frames_buffer: frames_buffer} + + # A single frame can have multiple commands, and each might have multiple responses. + # So we first handle each received command, and only then we 'flush', or send, each + # command to the socket. This also would allow us to better test the 'handler' logic. + commands + |> Enum.reduce(conn, &Handler.handle_message(&2, &1)) + |> flush_commands() + |> handle_closing() + end - def handle_info({:heartbeat}, conn) do - conn = Handler.send_request(conn, :heartbeat, correlation_sum: 0) + def handle_info({key, _socket}, conn) when key in [:tcp_closed, :ssl_closed] do + if conn.state == :connecting do + Logger.warning( + "The connection was closed by the host, after the socket was already open, while running the authentication sequence. This could be caused by the server not having Stream Plugin active" + ) + end - Process.send_after(self(), {:heartbeat}, conn.options[:heartbeat] * 1000) + %{conn | close_reason: key} + |> handle_closing() + end - {:noreply, conn} - end + def handle_info({key, _socket, reason}, conn) when key in [:tcp_error, :ssl_error] do + %{conn | close_reason: reason} + |> handle_closing() + end - def handle_info({:tcp_closed, _socket}, conn) do - if conn.state == :connecting do - Logger.warning( - "The connection was closed by the host, after the socket was already open, while running the authentication sequence. This could be caused by the server not having Stream Plugin active" - ) - end + def handle_info({:heartbeat}, conn) do + Process.send_after(self(), {:heartbeat}, conn.options[:heartbeat] * 1000) - conn = %{conn | socket: nil, state: :closed} |> Handler.handle_closed(:tcp_closed) + conn = send_request(conn, :heartbeat, correlation_sum: 0) - {:noreply, conn, :hibernate} - end + {:noreply, conn} + end - def handle_info({:tcp_error, _socket, reason}, conn) do - conn = %{conn | socket: nil, state: :closed} |> Handler.handle_closed(reason) + def handle_info(:flush_request_buffer, %Connection{state: :closed} = conn) do + Logger.warning("Connection is closed. Ignoring flush buffer request.") + {:noreply, conn} + end - {:noreply, conn} - end + # I'm not really sure how to test this behavior at the moment. + def handle_info(:flush_request_buffer, %Connection{} = conn) do + # There is probably a better way to reprocess the buffer, but I'm not sure how to do at the moment. + conn = + :queue.fold( + fn + {:call, {action, from}}, conn -> + {:noreply, conn} = handle_call(action, from, conn) + conn + + {:cast, action}, conn -> + {:noreply, conn} = handle_cast(action, conn) + conn + end, + conn, + conn.request_buffer + ) + + {:noreply, %{conn | request_buffer: :queue.new()}} + end + + @impl GenServer + def handle_continue({:connect}, %Connection{state: :closed} = conn) do + Logger.debug("Connecting to server: #{conn.options[:host]}:#{conn.options[:port]}") - def handle_info(:flush_request_buffer, %Connection{state: :closed} = conn) do - Logger.warning("Connection is closed. Ignoring flush buffer request.") + with {:ok, conn} <- connect(conn) do + Logger.debug("Connection stablished. Initiating properties exchange.") + + conn = + conn + |> send_request(:peer_properties) + + {:noreply, conn} + else + _ -> + Logger.error("Failed to connect to #{conn.options[:host]}:#{conn.options[:port]}") {:noreply, conn} - end + end + end - # I'm not really sure how to test this behavior at the moment. - def handle_info(:flush_request_buffer, %Connection{} = conn) do - # There is probably a better way to reprocess the buffer, but I'm not sure how to do at the moment. - conn = - :queue.fold( - fn - {:call, {action, from}}, conn -> - {:noreply, conn} = handle_call(action, from, conn) - conn - - {:cast, action}, conn -> - {:noreply, conn} = handle_cast(action, conn) - conn - end, - conn, - conn.request_buffer - ) - - {:noreply, %{conn | request_buffer: :queue.new()}} - end + defp connect(%Connection{} = conn) do + with {:ok, socket} <- conn.transport.connect(conn.options) do + {:ok, %{conn | socket: socket, state: :connecting}} + end + end - @impl true - def handle_continue({:connect}, %Connection{state: :closed} = conn) do - Logger.debug("Connecting to server: #{conn.options[:host]}:#{conn.options[:port]}") - - with {:ok, socket} <- - :gen_tcp.connect(String.to_charlist(conn.options[:host]), conn.options[:port], [:binary, active: true]), - :ok <- :gen_tcp.controlling_process(socket, self()) do - Logger.debug("Connection stablished. Initiating properties exchange.") - - conn = - %{conn | socket: socket, state: :connecting} - |> Handler.send_request(:peer_properties) - - {:noreply, conn} - else - _ -> - Logger.error("Failed to connect to #{conn.options[:host]}:#{conn.options[:port]}") - {:noreply, conn} - end - end + defp handle_closing(%Connection{state: :closing} = conn) do + for request <- conn.connect_requests do + GenServer.reply(request, {:error, :closed}) + end + + if is_port(conn.socket) do + :ok = conn.transport.close(conn.socket) + end - defguard is_offset(offset) - when offset in [:first, :last, :next] or - (is_tuple(offset) and tuple_size(offset) == 2 and elem(offset, 0) in [:offset, :timestamp]) + for {client, _data} <- Map.values(conn.request_tracker) do + GenServer.reply(client, {:error, {:closed, conn.close_reason}}) end + + conn = %{conn | request_tracker: %{}, connect_requests: [], socket: nil, state: :closed, close_reason: nil} + + {:noreply, conn, :hibernate} + end + + defp handle_closing(conn), do: {:noreply, conn} + + defp send_request(%Connection{} = conn, command, opts \\ []) do + conn + |> Helpers.push(:request, command, opts) + |> flush_commands() + end + + defp send_response(%Connection{} = conn, command, opts) do + conn + |> Helpers.push(:response, command, opts) + |> flush_commands() + end + + defp flush_commands(%Connection{} = conn) do + conn = + :queue.fold( + fn + command, conn -> + send_command(conn, command) + end, + conn, + conn.commands_buffer + ) + + %{conn | commands_buffer: :queue.new()} + end + + defp send_command(%Connection{} = conn, {:request, command, opts}) do + {correlation_sum, opts} = Keyword.pop(opts, :correlation_sum, 1) + + frame = + conn + |> Message.new_request(command, opts) + |> Encoder.encode() + + :ok = conn.transport.send(conn.socket, frame) + + %{ + conn + | correlation_sequence: conn.correlation_sequence + correlation_sum + } + end + + defp send_command(%Connection{} = conn, {:response, command, opts}) do + frame = + conn + |> Message.new_response(command, opts) + |> Encoder.encode() + + :ok = conn.transport.send(conn.socket, frame) + + conn end end diff --git a/lib/connection/transport/ssl.ex b/lib/connection/transport/ssl.ex new file mode 100644 index 0000000..7928cf4 --- /dev/null +++ b/lib/connection/transport/ssl.ex @@ -0,0 +1,23 @@ +defmodule RabbitMQStream.Connection.Transport.SSL do + @behaviour RabbitMQStream.Connection.Transport + + def connect(options) do + with {:ok, socket} <- + :ssl.connect( + String.to_charlist(options[:host]), + options[:port], + Keyword.merge(options[:ssl_opts], binary: true, active: true) + ), + :ok <- :ssl.controlling_process(socket, self()) do + {:ok, socket} + end + end + + def close(socket) do + :ssl.close(socket) + end + + def send(socket, data) do + :ssl.send(socket, data) + end +end diff --git a/lib/connection/transport/tcp.ex b/lib/connection/transport/tcp.ex new file mode 100644 index 0000000..f73ec67 --- /dev/null +++ b/lib/connection/transport/tcp.ex @@ -0,0 +1,19 @@ +defmodule RabbitMQStream.Connection.Transport.TCP do + @behaviour RabbitMQStream.Connection.Transport + + def connect(options) do + with {:ok, socket} <- + :gen_tcp.connect(String.to_charlist(options[:host]), options[:port], [:binary, active: true]), + :ok <- :gen_tcp.controlling_process(socket, self()) do + {:ok, socket} + end + end + + def close(socket) do + :gen_tcp.close(socket) + end + + def send(socket, data) do + :gen_tcp.send(socket, data) + end +end diff --git a/lib/connection/transport/transport.ex b/lib/connection/transport/transport.ex new file mode 100644 index 0000000..4cdf311 --- /dev/null +++ b/lib/connection/transport/transport.ex @@ -0,0 +1,8 @@ +defmodule RabbitMQStream.Connection.Transport do + @moduledoc false + @type t :: module() + + @callback connect(options :: Keyword.t()) :: {:ok, any()} | {:error, any()} + @callback close(socket :: any()) :: :ok + @callback send(socket :: any(), data :: iodata()) :: :ok +end diff --git a/lib/consumer/consumer.ex b/lib/consumer/consumer.ex new file mode 100644 index 0000000..2209a8f --- /dev/null +++ b/lib/consumer/consumer.ex @@ -0,0 +1,254 @@ +defmodule RabbitMQStream.Consumer do + @moduledoc """ + Used to declare a Persistent Consumer module. It is able to process + chunks by implementing the `handle_chunk/1` or `handle_chunk/2` callbacks. + + # Usage + + defmodule MyApp.MyConsumer do + use RabbitMQStream.Consumer, + connection: MyApp.MyConnection, + stream_name: "my_stream", + initial_offset: :first + + @impl true + def handle_chunk(%RabbitMQStream.OsirisChunk{} = _chunk, _consumer) do + :ok + end + end + + + # Parameters + + * `:connection` - The connection module to use. This is required. + * `:stream_name` - The name of the stream to consume. This is required. + * `:initial_offset` - The initial offset. This is required. + * `:initial_credit` - The initial credit to request from the server. Defaults to `50_000`. + * `:offset_tracking` - Offset tracking strategies to use. Defaults to `[count: [store_after: 50]]`. + * `:flow_control` - Flow control strategy to use. Defaults to `[count: [credit_after: {:count, 1}]]`. + * `:offset_reference` - + * `:private` - Private data that can hold any value, and is passed to the `handle_chunk/2` callback. + * `:serializer` - The module to use to decode the message. Defaults to `__MODULE__`, + which means that the consumer will use the `decode!/1` callback to decode the message, which is implemented by default to return the message as is. + + * `:properties` - Define the properties of the subscription. Can only have one option at a time. + * `:single_active_consumer`: set to `true` to enable [single active consumer](https://blog.rabbitmq.com/posts/2022/07/rabbitmq-3-11-feature-preview-single-active-consumer-for-streams/) for this subscription. + * `:super_stream`: set to the name of the super stream the subscribed is a partition of. + * `:filter`: List of strings that define the value of the filter_key to match. + * `:match_unfiltered`: whether to return messages without any filter value or not. + + + # Offset Tracking + + The consumer is able to track its progress in the stream by storing its + latests offset in the stream. Check [Offset Tracking with RabbitMQ Streams(https://blog.rabbitmq.com/posts/2021/09/rabbitmq-streams-offset-tracking/) for more information on + how offset tracking works. + + The consumer can be configured to use different offset tracking strategies, + which decide when to store the offset in the stream. You can implement your + own strategy by implementing the `RabbitMQStream.Consumer.OffsetTracking` + behaviour, and passing it to the `:offset_tracking` option. It defaults to + `RabbitMQStream.Consumer.OffsetTracking.CountStrategy`, which stores the + offset after, by default, every 50_000 messages. + + # Flow Control + + The RabbitMQ Streams server requires that the consumer declares how many + messages it is able to process at a time. This is done by informing an amount + of 'credits' to the server. After every chunk is sent, one credit is consumed, + and the server will send messages only if there are credits available. + + We can configure the consumer to automatically request more credits based on + a strategy. By default it uses the `RabbitMQStream.Consumer.FlowControl.MessageCount`, + which requests 1 additional credit for every 1 processed chunk. Please check + the RabbitMQStream.Consumer.FlowControl.MessageCount module for more information. + + You can also call `RabbitMQStream.Consumer.credit/2` to manually add more + credits to the subscription, or implement your own strategy by implementing + the `RabbitMQStream.Consumer.FlowControl` behaviour, and passing + it to the `:flow_control` option. + + You can find more information on the [RabbitMQ Streams documentation](https://www.rabbitmq.com/stream.html#flow-control). + + If you want an external process to be fully in control of the flow control + of a consumer, you can set the `:flow_control` option to `false`. Then + you can call `RabbitMQStream.Consumer.credit/2` to manually add more + credits to the subscription. + + + # Configuration + + You can configure each consumer with: + + config :rabbitmq_stream, MyApp.MyConsumer, + connection: MyApp.MyConnection, + stream_name: "my_stream", + initial_offset: :first, + initial_credit: 50_000, + offset_tracking: [count: [store_after: 50]], + flow_control: [count: [credit_after: {:count, 1}]], + serializer: Jason + + These options are overriden by the options passed to the `use` macro, which + are overriden by the options passed to `start_link/1`. + + And also you can override the defaults of all consumers with: + + config :rabbitmq_stream, :defaults, + consumer: [ + connection: MyApp.MyConnection, + initial_credit: 50_000, + # ... + ], + + Globally configuring all consumers ignores the following options: + + * `:stream_name` + * `:offset_reference` + * `:private` + + """ + defmacro __using__(opts) do + defaults = Application.get_env(:rabbitmq_stream, :defaults, []) + + serializer = Keyword.get(opts, :serializer, Keyword.get(defaults, :serializer)) + + quote location: :keep do + @opts unquote(opts) + require Logger + + @behaviour RabbitMQStream.Consumer + + def start_link(opts \\ []) do + unless !Keyword.has_key?(opts, :serializer) do + Logger.warning("You can only pass `:serializer` option to compile-time options.") + end + + opts = + Application.get_env(:rabbitmq_stream, __MODULE__, []) + |> Keyword.merge(@opts) + |> Keyword.merge(opts) + |> Keyword.put_new(:consumer_module, __MODULE__) + |> Keyword.put(:name, __MODULE__) + + RabbitMQStream.Consumer.start_link(opts) + end + + def child_spec(opts) do + %{id: __MODULE__, start: {__MODULE__, :start_link, [opts]}} + end + + def credit(amount) do + GenServer.cast(__MODULE__, {:credit, amount}) + end + + def get_credits() do + GenServer.call(__MODULE__, :get_credits) + end + + def before_start(_opts, state), do: state + + unquote( + if serializer != nil do + quote do + def decode!(message), do: unquote(serializer).decode!(message) + end + else + quote do + def decode!(message), do: message + end + end + ) + + defoverridable RabbitMQStream.Consumer + end + end + + def start_link(opts \\ []) do + opts = + Application.get_env(:rabbitmq_stream, :defaults, []) + |> Keyword.get(:consumer, []) + |> Keyword.drop([:stream_name, :offset_reference, :private]) + |> Keyword.merge(opts) + + GenServer.start_link(RabbitMQStream.Consumer.LifeCycle, opts, name: opts[:name]) + end + + def child_spec(opts) do + %{id: __MODULE__, start: {__MODULE__, :start_link, [opts]}} + end + + @optional_callbacks handle_chunk: 1, handle_chunk: 2, decode!: 1, handle_update: 2, before_start: 2 + + @doc """ + The callback that is invoked when a chunk is received. + + Each chunk contains a list of potentially many data entries, along with + metadata about the chunk itself. The callback is invoked once for each + chunk received. + + Optionally if you implement `handle_chunk/2`, it also passes the current + state of the consumer. It can be used to access the `private` field + passed to `start_link/1`, and other fields. + + The return value is ignored. + """ + @callback handle_chunk(chunk :: RabbitMQStream.OsirisChunk.t()) :: term() + @callback handle_chunk(chunk :: RabbitMQStream.OsirisChunk.t(), state :: t()) :: term() + + @callback handle_update(consumer :: t(), flag :: boolean()) :: + {:ok, RabbitMQStream.Connection.offset()} | {:error, any()} + + @callback decode!(message :: String.t()) :: term() + + @callback before_start(opts(), t()) :: t() + + defstruct [ + :offset_reference, + :connection, + :stream_name, + :offset_tracking, + :flow_control, + :id, + :last_offset, + # We could have delegated the tracking of the credit to the strategy, + # by adding declaring a callback similar to `after_chunk/3`. But it seems + # reasonable to have a `credit` function to manually add more credits, + # which would them possibly cause the strategy to not work as expected. + :credits, + :initial_credit, + :initial_offset, + :private, + :properties, + :consumer_module + ] + + @type t :: %__MODULE__{ + offset_reference: String.t(), + connection: GenServer.server(), + stream_name: String.t(), + id: non_neg_integer() | nil, + offset_tracking: [{RabbitMQStream.Consumer.OffsetTracking.t(), term()}], + flow_control: {RabbitMQStream.Consumer.FlowControl.t(), term()}, + last_offset: non_neg_integer() | nil, + private: any(), + credits: non_neg_integer(), + initial_credit: non_neg_integer(), + initial_offset: RabbitMQStream.Connection.offset(), + properties: [RabbitMQStream.Message.Types.ConsumerequestData.property()], + consumer_module: module() + } + + @type consumer_option :: + {:offset_reference, String.t()} + | {:connection, GenServer.server()} + | {:stream_name, String.t()} + | {:initial_offset, RabbitMQStream.Connection.offset()} + | {:initial_credit, non_neg_integer()} + | {:offset_tracking, [{RabbitMQStream.Consumer.OffsetTracking.t(), term()}]} + | {:flow_control, {RabbitMQStream.Consumer.FlowControl.t(), term()}} + | {:private, any()} + | {:properties, [RabbitMQStream.Message.Types.ConsumerequestData.property()]} + + @type opts :: [consumer_option()] +end diff --git a/lib/subscriber/credit/strategy.ex b/lib/consumer/flow_control/flow_control.ex similarity index 61% rename from lib/subscriber/credit/strategy.ex rename to lib/consumer/flow_control/flow_control.ex index 5079b6d..6e179b1 100644 --- a/lib/subscriber/credit/strategy.ex +++ b/lib/consumer/flow_control/flow_control.ex @@ -1,21 +1,21 @@ -defmodule RabbitMQStream.Subscriber.FlowControl.Strategy do +defmodule RabbitMQStream.Consumer.FlowControl do @type t :: module() @moduledoc """ Behavior for flow control strategies. - ## Existing Strategies + # Existing Strategies You can use the default strategies by passing a shorthand alias: - * `count` : `RabbitMQStream.Subscriber.FlowControl.MessageCount` + * `count` : `RabbitMQStream.Consumer.FlowControl.MessageCount` """ @doc """ Initializes the strategy state. - ## Parameters - * `opts` - a keyword list of the options passed to the subscriber, + # Parameters + * `opts` - a keyword list of the options passed to the consumer, merged with the options passed to the strategy itself. """ @callback init(opts :: term()) :: term() @@ -23,39 +23,39 @@ defmodule RabbitMQStream.Subscriber.FlowControl.Strategy do @doc """ Callback responsible for deciding whether to add more credit, based on its internal state. - ## Parameters + # Parameters * `state` - the state of the strategy * `subscription` - the state of the owner subscription process """ - @callback run(state :: term(), subscription :: RabbitMQStream.Subscriber.t()) :: + @callback run(state :: term(), subscription :: RabbitMQStream.Consumer.t()) :: {:credit, amount :: non_neg_integer(), state :: term()} | {:skip, state :: term()} @defaults %{ - count: RabbitMQStream.Subscriber.FlowControl.MessageCount + count: RabbitMQStream.Consumer.FlowControl.MessageCount } @doc false - def init([{strategy, opts}], subscriber_opts) do + def init([{strategy, opts}], consumer_opts) do strategy = @defaults[strategy] || strategy - {strategy, strategy.init(Keyword.merge(subscriber_opts, opts))} + {strategy, strategy.init(Keyword.merge(consumer_opts, opts))} end def init(false, _) do false end - def init(strategy, subscriber_opts) do + def init(strategy, consumer_opts) do strategy = @defaults[strategy] || strategy - {strategy, strategy.init(subscriber_opts)} + {strategy, strategy.init(consumer_opts)} end @doc false - def run(%RabbitMQStream.Subscriber{flow_control: {strategy, flow_state}} = state) do + def run(%RabbitMQStream.Consumer{flow_control: {strategy, flow_state}} = state) do case strategy.run(flow_state, state) do {:credit, amount, new_flow_control} -> - state.connection.credit(state.id, amount) + RabbitMQStream.Connection.credit(state.connection, state.id, amount) %{state | flow_control: {strategy, new_flow_control}, credits: state.credits + amount} {:skip, new_flow_control} -> @@ -63,5 +63,5 @@ defmodule RabbitMQStream.Subscriber.FlowControl.Strategy do end end - def run(%RabbitMQStream.Subscriber{flow_control: false} = state), do: state + def run(%RabbitMQStream.Consumer{flow_control: false} = state), do: state end diff --git a/lib/subscriber/credit/message_count.ex b/lib/consumer/flow_control/message_count.ex similarity index 77% rename from lib/subscriber/credit/message_count.ex rename to lib/consumer/flow_control/message_count.ex index 26d4db0..5ecdec6 100644 --- a/lib/subscriber/credit/message_count.ex +++ b/lib/consumer/flow_control/message_count.ex @@ -1,25 +1,25 @@ -defmodule RabbitMQStream.Subscriber.FlowControl.MessageCount do - @behaviour RabbitMQStream.Subscriber.FlowControl.Strategy +defmodule RabbitMQStream.Consumer.FlowControl.MessageCount do + @behaviour RabbitMQStream.Consumer.FlowControl @moduledoc """ Message Count Strategy Adds credits after the amount of consumed credit reaches a certain threshold. - ## Usage - defmodule MyApp.MySubscriber do - alias RabbitMQStream.Subscriber.FlowControl + # Usage + defmodule MyApp.MyConsumer do + alias RabbitMQStream.Consumer.FlowControl - use RabbitMQStream.Subscriber, + use RabbitMQStream.Consumer, offset_tracking: [FlowControl.MessageCount, credit_after: {:count, 1}] @impl true - def handle_chunk(_chunk, _subscriber) do + def handle_chunk(_chunk, _consumer) do :ok end end - ## Parameters + # Parameters * `credit_after` - The type of computation performed to decide whether to add more credit. Can be one of: diff --git a/lib/consumer/lifecycle.ex b/lib/consumer/lifecycle.ex new file mode 100644 index 0000000..db67d56 --- /dev/null +++ b/lib/consumer/lifecycle.ex @@ -0,0 +1,176 @@ +defmodule RabbitMQStream.Consumer.LifeCycle do + @moduledoc false + alias RabbitMQStream.Message.Request + alias RabbitMQStream.Consumer.{FlowControl, OffsetTracking} + + use GenServer + require Logger + + @impl true + def init(opts \\ []) do + opts = + opts + |> Keyword.put_new(:initial_credit, 50_000) + |> Keyword.put_new(:offset_tracking, count: [store_after: 50]) + |> Keyword.put_new(:flow_control, count: [credit_after: {:count, 1}]) + |> Keyword.put_new(:offset_reference, Atom.to_string(opts[:consumer_module])) + |> Keyword.put_new(:properties, []) + + unless opts[:initial_offset] != nil do + raise "initial_offset is required" + end + + # Prevent startup if 'single_active_consumer' is active, but there is no + # handle_update/2 callback defined. + if Keyword.get(opts[:properties], :single_active_consumer) != nil do + if not function_exported?(opts[:consumer_module], :handle_update, 2) do + raise "handle_update/2 must be implemented when using single-active-consumer property" + end + end + + opts = + opts + |> Keyword.put(:offset_tracking, OffsetTracking.init(opts[:offset_tracking], opts)) + |> Keyword.put(:flow_control, FlowControl.init(opts[:flow_control], opts)) + |> Keyword.put(:credits, opts[:initial_credit]) + + state = struct(RabbitMQStream.Consumer, opts) + + {:ok, state, {:continue, {:init, opts}}} + end + + @impl true + def handle_continue({:init, opts}, state) do + state = apply(state.consumer_module, :before_start, [opts, state]) + + last_offset = + case RabbitMQStream.Connection.query_offset(state.connection, state.stream_name, state.offset_reference) do + {:ok, offset} -> + {:offset, offset} + + _ -> + opts[:initial_offset] + end + + case RabbitMQStream.Connection.subscribe( + state.connection, + state.stream_name, + self(), + last_offset, + state.initial_credit, + state.properties + ) do + {:ok, id} -> + last_offset = + case last_offset do + {:offset, offset} -> + offset + + _ -> + nil + end + + {:noreply, %{state | id: id, last_offset: last_offset}} + + err -> + {:stop, err, state} + end + end + + @impl true + def terminate(_reason, %{id: nil}), do: :ok + + def terminate(_reason, state) do + # While not guaranteed, we attempt to store the offset when terminating. Useful for when performing + # upgrades, and in a 'single-active-consumer' scenario. + if state.last_offset != nil do + RabbitMQStream.Connection.store_offset( + state.connection, + state.stream_name, + state.offset_reference, + state.last_offset + ) + end + + RabbitMQStream.Connection.unsubscribe(state.connection, state.id) + :ok + end + + @impl true + def handle_info({:chunk, %RabbitMQStream.OsirisChunk{} = chunk}, state) do + # TODO: Possibly add 'filter_value', as described as necessary in the documentation. + chunk = RabbitMQStream.OsirisChunk.decode_messages!(chunk, state.consumer_module) + + cond do + function_exported?(state.consumer_module, :handle_chunk, 1) -> + apply(state.consumer_module, :handle_chunk, [chunk]) + + function_exported?(state.consumer_module, :handle_chunk, 2) -> + apply(state.consumer_module, :handle_chunk, [chunk, state]) + + true -> + raise "handle_chunk/1 or handle_chunk/2 must be implemented" + end + + offset_tracking = + for {strategy, track_state} <- state.offset_tracking do + if function_exported?(strategy, :after_chunk, 3) do + {strategy, strategy.after_chunk(track_state, chunk, state)} + else + {strategy, track_state} + end + end + + state = + %{ + state + | offset_tracking: offset_tracking, + last_offset: chunk.chunk_id, + credits: state.credits - chunk.num_entries + } + + state = state |> OffsetTracking.run() |> FlowControl.run() + + {:noreply, state} + end + + def handle_info(:run_offset_tracking, state) do + {:noreply, OffsetTracking.run(state)} + end + + def handle_info(:run_flow_control, state) do + {:noreply, FlowControl.run(state)} + end + + def handle_info({:command, %Request{command: :consumer_update} = request}, state) do + if function_exported?(state.consumer_module, :handle_update, 2) do + case apply(state.consumer_module, :handle_update, [state, request.data.active]) do + {:ok, offset} -> + Logger.debug("Consumer upgraded to active consumer") + RabbitMQStream.Connection.respond(state.connection, request, offset: offset, code: :ok) + {:noreply, state} + + {:error, reason} -> + Logger.error("Error updating consumer: #{inspect(reason)}") + RabbitMQStream.Connection.respond(state.connection, request, code: :internal_error) + + {:noreply, state} + end + else + Logger.error("handle_update/2 must be implemented when using single-active-consumer property") + RabbitMQStream.Connection.respond(state.connection, request, code: :internal_error) + {:noreply, state} + end + end + + @impl true + def handle_cast({:credit, amount}, state) do + RabbitMQStream.Connection.credit(state.connection, state.id, amount) + {:noreply, %{state | credits: state.credits + amount}} + end + + @impl true + def handle_call(:get_credits, _from, state) do + {:reply, state.credits, state} + end +end diff --git a/lib/subscriber/offset_tracking/interval.ex b/lib/consumer/offset_tracking/interval.ex similarity index 79% rename from lib/subscriber/offset_tracking/interval.ex rename to lib/consumer/offset_tracking/interval.ex index 7668803..3cd6975 100644 --- a/lib/subscriber/offset_tracking/interval.ex +++ b/lib/consumer/offset_tracking/interval.ex @@ -1,5 +1,5 @@ -defmodule RabbitMQStream.Subscriber.OffsetTracking.IntervalStrategy do - @behaviour RabbitMQStream.Subscriber.OffsetTracking.Strategy +defmodule RabbitMQStream.Consumer.OffsetTracking.IntervalStrategy do + @behaviour RabbitMQStream.Consumer.OffsetTracking @moduledoc """ Interval Strategy @@ -11,11 +11,11 @@ defmodule RabbitMQStream.Subscriber.OffsetTracking.IntervalStrategy do timer that may be running when the strategy is returns a `:store`. - ## Usage - defmodule MyApp.MySubscriber do - alias RabbitMQStream.Subscriber.OffsetTracking + # Usage + defmodule MyApp.MyConsumer do + alias RabbitMQStream.Consumer.OffsetTracking - use RabbitMQStream.Subscriber, + use RabbitMQStream.Consumer, offset_tracking: [OffsetTracking.IntervalStrategy, interval: 10_000] @impl true @@ -25,7 +25,7 @@ defmodule RabbitMQStream.Subscriber.OffsetTracking.IntervalStrategy do end - ## Parameters + # Parameters * `interval` - the time in milliseconds before storing the offset """ diff --git a/lib/subscriber/offset_tracking/strategy.ex b/lib/consumer/offset_tracking/offset_tracking.ex similarity index 69% rename from lib/subscriber/offset_tracking/strategy.ex rename to lib/consumer/offset_tracking/offset_tracking.ex index a006970..4be6b91 100644 --- a/lib/subscriber/offset_tracking/strategy.ex +++ b/lib/consumer/offset_tracking/offset_tracking.ex @@ -1,16 +1,16 @@ -defmodule RabbitMQStream.Subscriber.OffsetTracking.Strategy do +defmodule RabbitMQStream.Consumer.OffsetTracking do @type t :: module() @moduledoc """ Behavior for offset tracking strategies. - If you pass multiple strategies to the subscriber, which will be executed in order, and + If you pass multiple strategies to the consumer, which will be executed in order, and and halt after the first one that returns a `:store` request. - ## Existing Strategies + # Existing Strategies You can use the default strategies by passing a shorthand alias: - * `interval` : `RabbitMQStream.Subscriber.OffsetTracking.IntervalStrategy` - * `after` : `RabbitMQStream.Subscriber.OffsetTracking.CountStrategy` + * `interval` : `RabbitMQStream.Consumer.OffsetTracking.IntervalStrategy` + * `after` : `RabbitMQStream.Consumer.OffsetTracking.CountStrategy` """ @@ -19,8 +19,8 @@ defmodule RabbitMQStream.Subscriber.OffsetTracking.Strategy do @doc """ Initializes the strategy state. - ## Parameters - * `opts` - a keyword list of the options passed to the subscriber, + # Parameters + * `opts` - a keyword list of the options passed to the consumer, merged with the options passed to the strategy itself. """ @callback init(opts :: term()) :: term() @@ -34,24 +34,24 @@ defmodule RabbitMQStream.Subscriber.OffsetTracking.Strategy do @callback after_chunk( state :: term(), chunk :: RabbitMQStream.OsirisChunk.t(), - subscription :: RabbitMQStream.Subscriber.t() + subscription :: RabbitMQStream.Consumer.t() ) :: term() @doc """ Callback responsible for deciding whether to store the offset, based on its internal state. - ## Parameters + # Parameters * `state` - the state of the strategy * `subscription` - the state of the owner subscription process """ - @callback run(state :: term(), subscription :: RabbitMQStream.Subscriber.t()) :: + @callback run(state :: term(), subscription :: RabbitMQStream.Consumer.t()) :: {:store, state :: term()} | {:skip, state :: term()} @defaults %{ - count: RabbitMQStream.Subscriber.OffsetTracking.CountStrategy, - interval: RabbitMQStream.Subscriber.OffsetTracking.IntervalStrategy + count: RabbitMQStream.Consumer.OffsetTracking.CountStrategy, + interval: RabbitMQStream.Consumer.OffsetTracking.IntervalStrategy } @doc false def init(strategies, extra_opts \\ []) do @@ -71,9 +71,9 @@ defmodule RabbitMQStream.Subscriber.OffsetTracking.Strategy do end @doc false - def run(%RabbitMQStream.Subscriber{last_offset: nil} = state), do: state + def run(%RabbitMQStream.Consumer{last_offset: nil} = state), do: state - def run(%RabbitMQStream.Subscriber{} = state) do + def run(%RabbitMQStream.Consumer{} = state) do {_, offset_tracking} = Enum.reduce( state.offset_tracking, @@ -82,7 +82,12 @@ defmodule RabbitMQStream.Subscriber.OffsetTracking.Strategy do {strategy, track_state}, {:cont, acc} -> case strategy.run(track_state, state) do {:store, new_track_state} -> - state.connection.store_offset(state.stream_name, state.offset_reference, state.last_offset) + RabbitMQStream.Connection.store_offset( + state.connection, + state.stream_name, + state.offset_reference, + state.last_offset + ) {:halt, [{strategy, new_track_state} | acc]} diff --git a/lib/subscriber/offset_tracking/store_after_count.ex b/lib/consumer/offset_tracking/store_after_count.ex similarity index 74% rename from lib/subscriber/offset_tracking/store_after_count.ex rename to lib/consumer/offset_tracking/store_after_count.ex index b0ca885..7c6b34e 100644 --- a/lib/subscriber/offset_tracking/store_after_count.ex +++ b/lib/consumer/offset_tracking/store_after_count.ex @@ -1,16 +1,16 @@ -defmodule RabbitMQStream.Subscriber.OffsetTracking.CountStrategy do - @behaviour RabbitMQStream.Subscriber.OffsetTracking.Strategy +defmodule RabbitMQStream.Consumer.OffsetTracking.CountStrategy do + @behaviour RabbitMQStream.Consumer.OffsetTracking @moduledoc """ Count Strategy Stores the offset after every `store_after` messages. - ## Usage - defmodule MyApp.MySubscriber do - alias RabbitMQStream.Subscriber.OffsetTracking + # Usage + defmodule MyApp.MyConsumer do + alias RabbitMQStream.Consumer.OffsetTracking - use RabbitMQStream.Subscriber, + use RabbitMQStream.Consumer, offset_tracking: [OffsetTracking.CountStrategy, store_after: 50] @impl true @@ -20,7 +20,7 @@ defmodule RabbitMQStream.Subscriber.OffsetTracking.CountStrategy do end - ## Parameters + # Parameters * `store_after` - the number of messages to receive before storing the offset diff --git a/lib/message/buffer.ex b/lib/message/buffer.ex index 9d0e12b..3cb09f4 100644 --- a/lib/message/buffer.ex +++ b/lib/message/buffer.ex @@ -1,6 +1,5 @@ defmodule RabbitMQStream.Message.Buffer do @moduledoc false - alias RabbitMQStream.Message.Decoder # The code in this module is a one to one translation of the RabbitMQ decoding code at @@ -100,11 +99,13 @@ defmodule RabbitMQStream.Message.Buffer do # correctly insert them into the queue. # # This is different from the reference implementation, but necessary to - # maintain the order of commands received. + # maintain the order of commands received. The RabbitMQ's implementation + # might actually be wrong, since it doesn't seem to be used anywhere, and + # there are no tests for multiple frames in a single message. frames |> Enum.reverse() |> Enum.reduce(queue, fn frame, acc -> - :queue.in(Decoder.parse(frame), acc) + :queue.in(Decoder.decode(frame), acc) end) end end diff --git a/lib/message/data.ex b/lib/message/data.ex deleted file mode 100644 index 2db3efd..0000000 --- a/lib/message/data.ex +++ /dev/null @@ -1,410 +0,0 @@ -defmodule RabbitMQStream.Message.Data do - @moduledoc false - defmodule TuneData do - @moduledoc false - - defstruct [ - :frame_max, - :heartbeat - ] - end - - defmodule PeerPropertiesData do - @moduledoc false - - defstruct [ - :peer_properties - ] - end - - defmodule SaslHandshakeData do - @moduledoc false - - defstruct [:mechanisms] - end - - defmodule SaslAuthenticateData do - @moduledoc false - - defstruct [ - :mechanism, - :sasl_opaque_data - ] - end - - defmodule OpenData do - @moduledoc false - - defstruct [ - :vhost, - :connection_properties - ] - end - - defmodule HeartbeatData do - @moduledoc false - - defstruct [] - end - - defmodule CloseData do - @moduledoc false - - defstruct [ - :code, - :reason - ] - end - - defmodule CreateStreamData do - @moduledoc false - - defstruct [ - :stream_name, - :arguments - ] - end - - defmodule DeleteStreamData do - @moduledoc false - - defstruct [ - :stream_name - ] - end - - defmodule StoreOffsetData do - @moduledoc false - - defstruct [ - :offset_reference, - :stream_name, - :offset - ] - end - - defmodule QueryOffsetData do - @moduledoc false - - defstruct [ - :offset_reference, - :stream_name, - :offset - ] - end - - defmodule QueryMetadataData do - @moduledoc false - - defstruct [ - :brokers, - :streams - ] - end - - defmodule MetadataUpdateData do - @moduledoc false - - defstruct [ - :stream_name - ] - end - - defmodule DeclarePublisherData do - @moduledoc false - - defstruct [ - :id, - :publisher_reference, - :stream_name - ] - end - - defmodule DeletePublisherData do - @moduledoc false - - defstruct [ - :publisher_id - ] - end - - defmodule BrokerData do - @moduledoc false - - defstruct [ - :reference, - :host, - :port - ] - end - - defmodule StreamData do - @moduledoc false - - defstruct [ - :code, - :name, - :leader, - :replicas - ] - end - - defmodule QueryPublisherSequenceData do - @moduledoc false - - defstruct [ - :publisher_reference, - :stream_name, - :sequence - ] - end - - defmodule PublishData do - @moduledoc false - - defstruct [ - :publisher_id, - :published_messages - ] - end - - defmodule PublishErrorData do - @moduledoc false - - defmodule Error do - @moduledoc false - - defstruct [ - :publishing_id, - :code - ] - end - - defstruct [ - :publisher_id, - :errors - ] - end - - defmodule PublishConfirmData do - @moduledoc false - - defstruct [ - :publisher_id, - :publishing_ids - ] - end - - defmodule SubscribeRequestData do - @moduledoc false - # Supported properties: - - # * `single-active-consumer`: set to `true` to enable https://blog.rabbitmq.com/posts/2022/07/rabbitmq-3-11-feature-preview-single-active-consumer-for-streams/[single active consumer] for this subscription. - # * `super-stream`: set to the name of the super stream the subscribed is a partition of. - # * `filter.` (e.g. `filter.0`, `filter.1`, etc): prefix to use to define filter values for the subscription. - # * `match-unfiltered`: whether to return messages without any filter value or not. - - @type t :: %{ - subscription_id: non_neg_integer(), - stream_name: String.t(), - offset: RabbitMQStream.Connection.offset(), - credit: non_neg_integer(), - properties: %{String.t() => String.t()} - } - - defstruct [ - :subscription_id, - :stream_name, - :offset, - :credit, - :properties - ] - end - - defmodule UnsubscribeRequestData do - @moduledoc false - - @type t :: %{ - subscription_id: non_neg_integer() - } - - defstruct [ - :subscription_id - ] - end - - defmodule CreditRequestData do - @moduledoc false - - @type t :: %{ - subscription_id: non_neg_integer(), - credit: non_neg_integer() - } - - defstruct [ - :subscription_id, - :credit - ] - end - - defmodule SubscribeResponseData do - @moduledoc false - @type t :: %{} - defstruct [] - end - - defmodule UnsubscribeResponseData do - @moduledoc false - @type t :: %{} - defstruct [] - end - - defmodule CreditResponseData do - @moduledoc false - @type t :: %{} - defstruct [] - end - - defmodule RouteRequestData do - @moduledoc false - @enforce_keys [:routing_key, :super_stream] - @type t :: %{ - routing_key: String.t(), - super_stream: String.t() - } - defstruct [ - :routing_key, - :super_stream - ] - end - - defmodule RouteResponseData do - @moduledoc false - @enforce_keys [:stream] - @type t :: %{stream: String.t()} - defstruct [:stream] - end - - defmodule PartitionsQueryRequestData do - @moduledoc false - @enforce_keys [:super_stream] - @type t :: %{super_stream: String.t()} - defstruct [:super_stream] - end - - defmodule PartitionsQueryRequestData do - @moduledoc false - @enforce_keys [:stream] - @type t :: %{stream: String.t()} - defstruct [:stream] - end - - defmodule DeliverData do - @moduledoc false - @enforce_keys [:subscription_id, :osiris_chunk] - @type t :: %{ - committed_offset: non_neg_integer() | nil, - subscription_id: non_neg_integer(), - osiris_chunk: RabbitMQStream.OsirisChunk.t() - } - defstruct [ - :committed_offset, - :subscription_id, - :osiris_chunk - ] - end - - def decode_data(:close, ""), do: %CloseData{} - def decode_data(:create_stream, ""), do: %CreateStreamData{} - def decode_data(:delete_stream, ""), do: %DeleteStreamData{} - def decode_data(:declare_publisher, ""), do: %DeclarePublisherData{} - def decode_data(:delete_publisher, ""), do: %DeletePublisherData{} - def decode_data(:subscribe, ""), do: %SubscribeResponseData{} - def decode_data(:unsubscribe, ""), do: %UnsubscribeResponseData{} - def decode_data(:credit, ""), do: %CreditResponseData{} - - def decode_data(:query_offset, <>) do - %QueryOffsetData{offset: offset} - end - - def decode_data(:query_publisher_sequence, <>) do - %QueryPublisherSequenceData{sequence: sequence} - end - - def decode_data(:peer_properties, buffer) do - {"", peer_properties} = - decode_array(buffer, fn buffer, acc -> - {buffer, key} = fetch_string(buffer) - {buffer, value} = fetch_string(buffer) - - {buffer, [{key, value} | acc]} - end) - - %PeerPropertiesData{peer_properties: peer_properties} - end - - def decode_data(:sasl_handshake, buffer) do - {"", mechanisms} = - decode_array(buffer, fn buffer, acc -> - {buffer, value} = fetch_string(buffer) - {buffer, [value | acc]} - end) - - %SaslHandshakeData{mechanisms: mechanisms} - end - - def decode_data(:sasl_authenticate, buffer) do - %SaslAuthenticateData{sasl_opaque_data: buffer} - end - - def decode_data(:tune, <>) do - %TuneData{frame_max: frame_max, heartbeat: heartbeat} - end - - def decode_data(:open, buffer) do - connection_properties = - if buffer != "" do - {"", connection_properties} = - decode_array(buffer, fn buffer, acc -> - {buffer, key} = fetch_string(buffer) - {buffer, value} = fetch_string(buffer) - - {buffer, [{key, value} | acc]} - end) - - connection_properties - else - [] - end - - %OpenData{connection_properties: connection_properties} - end - - def fetch_string(<>) do - {rest, to_string(text)} - end - - def decode_array("", _) do - {"", []} - end - - def decode_array(<<0::integer-size(32), buffer::binary>>, _) do - {buffer, []} - end - - def decode_array(<>, foo) do - Enum.reduce(0..(size - 1), {buffer, []}, fn _, {buffer, acc} -> - foo.(buffer, acc) - end) - end - - def decode_array(<<0::integer-size(32), buffer::binary>>, _) do - {buffer, []} - end - - def decode_array(<>, foo) do - Enum.reduce(0..(size - 1), {buffer, []}, fn _, {buffer, acc} -> - foo.(buffer, acc) - end) - end -end diff --git a/lib/message/data/data.ex b/lib/message/data/data.ex new file mode 100644 index 0000000..e6a24db --- /dev/null +++ b/lib/message/data/data.ex @@ -0,0 +1,495 @@ +defmodule RabbitMQStream.Message.Data do + alias RabbitMQStream.Message.Types + alias RabbitMQStream.Message.{Response, Request} + import RabbitMQStream.Message.Helpers + + def decode(%{command: :heartbeat}, ""), do: %Types.HeartbeatData{} + def decode(%Response{command: :close}, ""), do: %Types.CloseResponseData{} + + def decode(%Response{command: :create_stream}, ""), do: %Types.CreateStreamResponseData{} + def decode(%Response{command: :delete_stream}, ""), do: %Types.DeleteStreamResponseData{} + def decode(%Response{command: :declare_producer}, ""), do: %Types.DeclareProducerResponseData{} + def decode(%Response{command: :delete_producer}, ""), do: %Types.DeleteProducerResponseData{} + def decode(%Response{command: :subscribe}, ""), do: %Types.SubscribeResponseData{} + def decode(%Response{command: :unsubscribe}, ""), do: %Types.UnsubscribeResponseData{} + def decode(%Response{command: :credit}, ""), do: %Types.CreditResponseData{} + def decode(%Response{command: :store_offset}, ""), do: %Types.StoreOffsetResponseData{} + def decode(%Response{command: :create_super_stream}, ""), do: %Types.CreateSuperStreamResponseData{} + def decode(%Response{command: :delete_super_stream}, ""), do: %Types.DeleteSuperStreamResponseData{} + + def decode(%Request{command: :publish_confirm}, buffer) do + <> = buffer + + {"", publishing_ids} = + decode_array(buffer, fn buffer, acc -> + <> = buffer + {buffer, [publishing_id] ++ acc} + end) + + %Types.PublishConfirmData{producer_id: producer_id, publishing_ids: publishing_ids} + end + + def decode(%Response{command: :publish_error}, buffer) do + <> = buffer + + {"", errors} = + decode_array(buffer, fn buffer, acc -> + << + publishing_id::unsigned-integer-size(64), + code::unsigned-integer-size(16), + buffer::binary + >> = buffer + + entry = %Types.PublishErrorData.Error{ + code: decode_code(code), + publishing_id: publishing_id + } + + {buffer, [entry] ++ acc} + end) + + %Types.PublishErrorData{producer_id: producer_id, errors: errors} + end + + def decode(%Request{version: 1, command: :deliver}, buffer) do + <> = buffer + + osiris_chunk = RabbitMQStream.OsirisChunk.decode!(rest) + + %Types.DeliverData{subscription_id: subscription_id, osiris_chunk: osiris_chunk} + end + + def decode(%Request{version: 2, command: :deliver}, buffer) do + <> = buffer + + osiris_chunk = RabbitMQStream.OsirisChunk.decode!(rest) + + %Types.DeliverData{ + subscription_id: subscription_id, + committed_offset: committed_offset, + osiris_chunk: osiris_chunk + } + end + + def decode(%{command: :query_metadata}, buffer) do + {buffer, brokers} = + decode_array(buffer, fn buffer, acc -> + <> = buffer + + <> = buffer + + <> = buffer + + data = %Types.QueryMetadataResponseData.BrokerData{ + reference: reference, + host: host, + port: port + } + + {buffer, [data] ++ acc} + end) + + {"", streams} = + decode_array(buffer, fn buffer, acc -> + << + size::integer-size(16), + name::binary-size(size), + code::unsigned-integer-size(16), + leader::unsigned-integer-size(16), + buffer::binary + >> = buffer + + {buffer, replicas} = + decode_array(buffer, fn buffer, acc -> + <> = buffer + + {buffer, [replica] ++ acc} + end) + + data = %Types.QueryMetadataResponseData.StreamData{ + code: decode_code(code), + name: name, + leader: leader, + replicas: replicas + } + + {buffer, [data] ++ acc} + end) + + %Types.QueryMetadataResponseData{brokers: brokers, streams: streams} + end + + def decode(%Request{command: :close}, <>) do + {"", reason} = decode_string(buffer) + + %Types.CloseRequestData{code: code, reason: reason} + end + + def decode(%{command: :metadata_update}, <>) do + {"", stream_name} = decode_string(buffer) + + %Types.MetadataUpdateData{stream_name: stream_name, code: code} + end + + def decode(%{command: :query_offset}, <>) do + %Types.QueryOffsetResponseData{offset: offset} + end + + def decode(%Response{command: :query_producer_sequence}, <>) do + %Types.QueryProducerSequenceResponseData{sequence: sequence} + end + + def decode(%{command: :peer_properties}, buffer) do + {"", peer_properties} = + decode_array(buffer, fn buffer, acc -> + {buffer, key} = decode_string(buffer) + {buffer, value} = decode_string(buffer) + + {buffer, [{key, value} | acc]} + end) + + %Types.PeerPropertiesData{peer_properties: Map.new(peer_properties)} + end + + def decode(%{command: :sasl_handshake}, buffer) do + {"", mechanisms} = + decode_array(buffer, fn buffer, acc -> + {buffer, value} = decode_string(buffer) + {buffer, [value | acc]} + end) + + %Types.SaslHandshakeData{mechanisms: mechanisms} + end + + def decode(%{command: :sasl_authenticate}, buffer) do + %Types.SaslAuthenticateData{sasl_opaque_data: buffer} + end + + def decode(%{command: :tune}, <>) do + %Types.TuneData{frame_max: frame_max, heartbeat: heartbeat} + end + + def decode(%{command: :open}, buffer) do + {"", connection_properties} = + decode_array(buffer, fn buffer, acc -> + {buffer, key} = decode_string(buffer) + {buffer, value} = decode_string(buffer) + + {buffer, [{key, value} | acc]} + end) + + %Types.OpenResponseData{connection_properties: connection_properties} + end + + def decode(%Response{command: :route}, buffer) do + {"", streams} = + decode_array(buffer, fn buffer, acc -> + {buffer, value} = decode_string(buffer) + {buffer, [value | acc]} + end) + + %Types.RouteResponseData{streams: streams} + end + + def decode(%Response{command: :partitions}, buffer) do + {"", streams} = + decode_array(buffer, fn buffer, acc -> + {buffer, value} = decode_string(buffer) + {buffer, [value | acc]} + end) + + %Types.PartitionsQueryResponseData{streams: streams} + end + + def decode(%Response{command: :exchange_command_versions}, buffer) do + {"", commands} = + decode_array(buffer, fn buffer, acc -> + << + key::unsigned-integer-size(16), + min_version::unsigned-integer-size(16), + max_version::unsigned-integer-size(16), + rest::binary + >> = buffer + + value = %Types.ExchangeCommandVersionsData.Command{ + key: decode_command(key), + min_version: min_version, + max_version: max_version + } + + {rest, [value | acc]} + end) + + %Types.ExchangeCommandVersionsData{commands: commands} + end + + def decode(%Response{command: :consumer_update}, buffer) do + {"", offset} = decode_offset(buffer) + + %Types.ConsumerUpdateResponseData{offset: offset} + end + + def decode(%Request{command: :consumer_update}, buffer) do + <> = buffer + + %Types.ConsumerUpdateRequestData{subscription_id: subscription_id, active: flag == 1} + end + + def decode(%Response{command: :stream_stats}, buffer) do + {"", stats} = + decode_array(buffer, fn buffer, acc -> + {buffer, key} = decode_string(buffer) + <> = buffer + + {buffer, [{key, value} | acc]} + end) + + %Types.StreamStatsResponseData{stats: Map.new(stats)} + end + + def encode(%Response{command: :close}) do + <<>> + end + + def encode(%Request{command: :peer_properties, data: data}) do + properties = encode_map(data.peer_properties) + + <> + end + + def encode(%Request{command: :sasl_handshake}) do + <<>> + end + + def encode(%Request{command: :sasl_authenticate, data: data}) do + mechanism = encode_string(data.mechanism) + + credentials = + encode_bytes("\u0000#{data.sasl_opaque_data[:username]}\u0000#{data.sasl_opaque_data[:password]}") + + <> + end + + def encode(%Request{command: :open, data: data}) do + vhost = encode_string(data.vhost) + + <> + end + + def encode(%Request{command: :heartbeat}) do + <<>> + end + + def encode(%Request{command: :tune, data: data}) do + <> + end + + def encode(%Request{command: :close, data: data}) do + reason = encode_string(data.reason) + + <> + end + + def encode(%Request{command: :create_stream, data: data}) do + stream_name = encode_string(data.stream_name) + arguments = encode_map(data.arguments) + + <> + end + + def encode(%Request{command: :delete_stream, data: data}) do + stream_name = encode_string(data.stream_name) + + <> + end + + def encode(%Request{command: :store_offset, data: data}) do + offset_reference = encode_string(data.offset_reference) + stream_name = encode_string(data.stream_name) + + << + offset_reference::binary, + stream_name::binary, + data.offset::unsigned-integer-size(64) + >> + end + + def encode(%Request{command: :query_offset, data: data}) do + offset_reference = encode_string(data.offset_reference) + stream_name = encode_string(data.stream_name) + + << + offset_reference::binary, + stream_name::binary + >> + end + + def encode(%Request{command: :declare_producer, data: data}) do + producer_reference = encode_string(data.producer_reference) + stream_name = encode_string(data.stream_name) + + << + data.id::unsigned-integer-size(8), + producer_reference::binary, + stream_name::binary + >> + end + + def encode(%Request{command: :delete_producer, data: data}) do + <> + end + + def encode(%Request{command: :query_metadata, data: data}) do + streams = + data.streams + |> Enum.map(&encode_string/1) + |> encode_array() + + <> + end + + def encode(%Request{command: :query_producer_sequence, data: data}) do + producer_reference = encode_string(data.producer_reference) + stream_name = encode_string(data.stream_name) + + <> + end + + def encode(%Request{version: 1, command: :publish, data: data}) do + messages = + encode_array( + for {publishing_id, message, nil} <- data.messages do + << + publishing_id::unsigned-integer-size(64), + encode_bytes(message)::binary + >> + end + ) + + <> + end + + def encode(%Request{version: 2, command: :publish, data: data}) do + messages = + encode_array( + for {publishing_id, message, filter_value} <- data.messages do + << + publishing_id::unsigned-integer-size(64), + encode_string(filter_value)::binary, + encode_bytes(message)::binary + >> + end + ) + + <> + end + + def encode(%Request{command: :subscribe, data: data}) do + stream_name = encode_string(data.stream_name) + + offset = encode_offset(data.offset) + + properties = + Enum.reduce(data.properties, [], fn + {:filter, entries}, acc -> + filter = + for {entry, i} <- Enum.with_index(entries) do + {"filter.#{i}", entry} + end + + filter ++ acc + + {:single_active_consumer, name}, acc -> + [{"single-active-consumer", true}, {"name", name} | acc] + + {:super_stream, name}, acc -> + [{"super-stream", name} | acc] + + {key, value}, acc -> + [{String.replace("#{key}", "_", "-"), value} | acc] + end) + |> encode_map() + + << + data.subscription_id::unsigned-integer-size(8), + stream_name::binary, + offset::binary, + data.credit::unsigned-integer-size(16), + properties::binary + >> + end + + def encode(%Request{command: :unsubscribe, data: data}) do + <> + end + + def encode(%Request{command: :credit, data: data}) do + << + data.subscription_id::unsigned-integer-size(8), + data.credit::unsigned-integer-size(16) + >> + end + + def encode(%Request{command: :route, data: data}) do + routing_key = encode_string(data.routing_key) + super_stream = encode_string(data.super_stream) + + <> + end + + def encode(%Request{command: :partitions, data: data}) do + super_stream = encode_string(data.super_stream) + + <> + end + + def encode(%Request{command: :exchange_command_versions, data: data}) do + encode_array( + for command <- data.commands do + << + encode_command(command.key)::unsigned-integer-size(16), + command.min_version::unsigned-integer-size(16), + command.max_version::unsigned-integer-size(16) + >> + end + ) + end + + def encode(%Response{command: :tune, data: data}) do + << + data.frame_max::unsigned-integer-size(32), + data.heartbeat::unsigned-integer-size(32) + >> + end + + def encode(%Response{command: :consumer_update, data: data, code: :ok}) do + encode_offset(data.offset) + end + + def encode(%Response{command: :consumer_update}) do + # The server expects an offset even if the response is not :ok. + # So we send a default one. + encode_offset(:next) + end + + def encode(%Request{command: :stream_stats, data: data}) do + encode_string(data.stream_name) + end + + def encode(%Request{command: :create_super_stream, data: data}) do + name = encode_string(data.name) + + {partitions, binding_keys} = Enum.unzip(data.partitions) + + partitions = encode_array(partitions, &encode_string/1) + binding_keys = encode_array(binding_keys, &encode_string/1) + + arguments = encode_map(data.arguments) + + <> + end + + def encode(%Request{command: :delete_super_stream, data: data}) do + encode_string(data.name) + end +end diff --git a/lib/message/data/types.ex b/lib/message/data/types.ex new file mode 100644 index 0000000..e3410f7 --- /dev/null +++ b/lib/message/data/types.ex @@ -0,0 +1,579 @@ +defmodule RabbitMQStream.Message.Types do + @moduledoc false + alias RabbitMQStream.Message.Helpers + + defmodule TuneData do + @moduledoc false + @enforce_keys [:frame_max, :heartbeat] + @type t :: %{ + frame_max: non_neg_integer(), + heartbeat: non_neg_integer() + } + + defstruct [ + :frame_max, + :heartbeat + ] + end + + defmodule PeerPropertiesData do + @moduledoc false + @enforce_keys [:peer_properties] + @type t :: %{peer_properties: [[String.t()]]} + + defstruct [:peer_properties] + end + + defmodule SaslHandshakeData do + @moduledoc false + + @type t :: %{mechanisms: [String.t()]} + + defstruct [:mechanisms] + end + + defmodule SaslAuthenticateData do + @moduledoc false + @type t :: %{ + mechanism: String.t(), + sasl_opaque_data: Keyword.t() + } + + defstruct [ + :mechanism, + :sasl_opaque_data + ] + end + + defmodule OpenRequestData do + @moduledoc false + @enforce_keys [:vhost] + @type t :: %{ + vhost: String.t() + } + + defstruct [ + :vhost + ] + end + + defmodule OpenResponseData do + @moduledoc false + @enforce_keys [:connection_properties] + @type t :: %{ + connection_properties: Keyword.t() + } + + defstruct [ + :connection_properties + ] + end + + defmodule HeartbeatData do + @moduledoc false + @type t :: %{} + + defstruct [] + end + + defmodule CloseRequestData do + @moduledoc false + @enforce_keys [:code, :reason] + + @type t :: %{ + code: RabbitMQStream.Message.Helpers.code(), + reason: String.t() + } + defstruct [ + :code, + :reason + ] + end + + defmodule CloseResponseData do + @moduledoc false + @type t :: %{} + defstruct [] + end + + defmodule CreateStreamRequestData do + @moduledoc false + @enforce_keys [:stream_name, :arguments] + @type t :: %{ + stream_name: String.t(), + arguments: Keyword.t() + } + + defstruct [ + :stream_name, + :arguments + ] + end + + defmodule CreateStreamResponseData do + @moduledoc false + + defstruct [] + end + + defmodule DeleteStreamRequestData do + @moduledoc false + @enforce_keys [:stream_name] + @type t :: %{stream_name: String.t()} + defstruct [:stream_name] + end + + defmodule DeleteStreamResponseData do + @moduledoc false + + @type t :: %{} + defstruct [] + end + + defmodule StoreOffsetRequestData do + @moduledoc false + + @enforce_keys [:stream_name, :offset_reference, :offset] + @type t :: %{ + stream_name: String.t(), + offset_reference: String.t(), + offset: non_neg_integer() + } + + defstruct [ + :offset_reference, + :stream_name, + :offset + ] + end + + defmodule StoreOffsetResponseData do + @moduledoc false + @type t :: %{} + defstruct [] + end + + defmodule QueryOffsetRequestData do + @moduledoc false + @enforce_keys [:stream_name, :offset_reference] + @type t :: %{ + stream_name: String.t(), + offset_reference: String.t() + } + + defstruct [:offset_reference, :stream_name] + end + + defmodule QueryOffsetResponseData do + @moduledoc false + + @enforce_keys [:offset] + @type t :: %{offset: non_neg_integer()} + defstruct [:offset] + end + + defmodule QueryMetadataRequestData do + @moduledoc false + @enforce_keys [:streams] + @type t :: %{offset: [String.t()]} + defstruct [:streams] + end + + defmodule QueryMetadataResponseData do + @moduledoc false + @type t :: %{ + streams: [StreamData.t()], + brokers: [BrokerData.t()] + } + + defstruct [ + :streams, + :brokers + ] + + defmodule BrokerData do + @moduledoc false + @enforce_keys [:reference, :host, :port] + @type t :: %{ + reference: non_neg_integer(), + host: String.t(), + port: non_neg_integer() + } + + defstruct [ + :reference, + :host, + :port + ] + end + + defmodule StreamData do + @moduledoc false + @enforce_keys [:code, :name, :leader, :replicas] + @type t :: %{ + code: RabbitMQStream.Message.Helpers.code(), + name: String.t(), + leader: non_neg_integer(), + replicas: [non_neg_integer()] + } + + defstruct [ + :code, + :name, + :leader, + :replicas + ] + end + end + + defmodule MetadataUpdateData do + @moduledoc false + @enforce_keys [:stream_name, :code] + @type t :: %{ + stream_name: String.t(), + code: non_neg_integer() + } + defstruct [:stream_name, :code] + end + + defmodule DeclareProducerRequestData do + @moduledoc false + @enforce_keys [:id, :producer_reference, :stream_name] + @type t :: %{ + id: non_neg_integer(), + producer_reference: String.t(), + stream_name: String.t() + } + + defstruct [ + :id, + :producer_reference, + :stream_name + ] + end + + defmodule DeclareProducerResponseData do + @moduledoc false + @type t :: %{} + defstruct [] + end + + defmodule DeleteProducerRequestData do + @moduledoc false + @enforce_keys [:producer_id] + @type t :: %{producer_id: non_neg_integer()} + defstruct [:producer_id] + end + + defmodule DeleteProducerResponseData do + @moduledoc false + @type t :: %{} + defstruct [] + end + + defmodule QueryProducerSequenceRequestData do + @moduledoc false + @enforce_keys [:producer_reference, :stream_name] + @type t :: %{ + producer_reference: String.t(), + stream_name: String.t() + } + + defstruct [:producer_reference, :stream_name] + end + + defmodule QueryProducerSequenceResponseData do + @moduledoc false + @enforce_keys [:sequence] + @type t :: %{sequence: non_neg_integer()} + defstruct [:sequence] + end + + defmodule PublishData do + @moduledoc false + @enforce_keys [:producer_id, :messages] + @type t :: %{ + producer_id: non_neg_integer(), + messages: [{publishing_id :: non_neg_integer(), message :: binary(), filter_value :: binary() | nil}] + } + + defstruct [:producer_id, :messages] + end + + defmodule PublishErrorData do + @moduledoc false + @enforce_keys [:producer_id, :errors] + @type t :: %{ + producer_id: non_neg_integer(), + errors: [Error.t()] + } + defstruct [ + :producer_id, + :errors + ] + + defmodule Error do + @moduledoc false + @enforce_keys [:publishing_id, :code] + @type t :: %{ + publishing_id: non_neg_integer(), + code: RabbitMQStream.Message.Helpers.code() + } + + defstruct [:publishing_id, :code] + end + end + + defmodule PublishConfirmData do + @moduledoc false + @enforce_keys [:producer_id, :publishing_ids] + @type t :: %{ + producer_id: non_neg_integer(), + publishing_ids: [non_neg_integer()] + } + + defstruct [:producer_id, :publishing_ids] + end + + defmodule SubscribeRequestData do + @moduledoc """ + Supported properties: + + * `single-active-consumer`: set to `true` to enable [single active consumer](https://blog.rabbitmq.com/posts/2022/07/rabbitmq-3-11-feature-preview-single-active-consumer-for-streams/) for this subscription. + * `super-stream`: set to the name of the super stream the subscribed is a partition of. + * `filter.` (e.g. `filter.0`, `filter.1`, etc): prefix to use to define filter values for the subscription. + * `match-unfiltered`: whether to return messages without any filter value or not. + """ + + defstruct [ + :subscription_id, + :stream_name, + :offset, + :credit, + :properties + ] + + @type t :: %{ + subscription_id: non_neg_integer(), + stream_name: String.t(), + offset: RabbitMQStream.Connection.offset(), + credit: non_neg_integer(), + properties: [property()] + } + + @type property :: + {:single_active_consumer, String.t()} + | {:super_stream, String.t()} + | {:filter, [String.t()]} + | {:match_unfiltered, boolean()} + + def new!(opts) do + %__MODULE__{ + credit: opts[:credit], + offset: opts[:offset], + properties: opts[:properties], + stream_name: opts[:stream_name], + subscription_id: opts[:subscription_id] + } + end + end + + defmodule ConsumerUpdateRequestData do + @moduledoc false + @enforce_keys [:subscription_id, :active] + + @type t :: %{ + subscription_id: non_neg_integer(), + active: boolean() + } + + defstruct [:subscription_id, :active] + end + + defmodule ConsumerUpdateResponseData do + @moduledoc false + @enforce_keys [:offset] + + @type t :: %{offset: RabbitMQStream.Connection.offset()} + + defstruct [:offset] + end + + defmodule UnsubscribeRequestData do + @moduledoc false + @enforce_keys [:subscription_id] + @type t :: %{subscription_id: non_neg_integer()} + defstruct [:subscription_id] + end + + defmodule CreditRequestData do + @moduledoc false + @enforce_keys [:subscription_id, :credit] + @type t :: %{ + subscription_id: non_neg_integer(), + credit: non_neg_integer() + } + defstruct [:subscription_id, :credit] + end + + defmodule SubscribeResponseData do + @moduledoc false + @type t :: %{} + defstruct [] + end + + defmodule UnsubscribeResponseData do + @moduledoc false + @type t :: %{} + defstruct [] + end + + defmodule CreditResponseData do + @moduledoc false + @type t :: %{} + defstruct [] + end + + defmodule RouteRequestData do + @moduledoc false + @enforce_keys [:routing_key, :super_stream] + @type t :: %{ + routing_key: String.t(), + super_stream: String.t() + } + defstruct [:routing_key, :super_stream] + end + + defmodule RouteResponseData do + @moduledoc false + @enforce_keys [:streams] + @type t :: %{streams: [String.t()]} + defstruct [:streams] + end + + defmodule PartitionsQueryRequestData do + @moduledoc false + @enforce_keys [:super_stream] + @type t :: %{super_stream: String.t()} + defstruct [:super_stream] + end + + defmodule PartitionsQueryResponseData do + @moduledoc false + @enforce_keys [:streams] + @type t :: %{streams: [String.t()]} + defstruct [:streams] + end + + defmodule DeliverData do + @moduledoc false + @enforce_keys [:subscription_id, :osiris_chunk] + @type t :: %{ + committed_offset: non_neg_integer() | nil, + subscription_id: non_neg_integer(), + osiris_chunk: RabbitMQStream.OsirisChunk.t() + } + defstruct [ + :committed_offset, + :subscription_id, + :osiris_chunk + ] + end + + defmodule ExchangeCommandVersionsData do + @moduledoc false + @enforce_keys [:commands] + @type t :: %{commands: [Command.t()]} + defstruct [:commands] + + defmodule Command do + @moduledoc false + @enforce_keys [:key, :min_version, :max_version] + @type t :: %{ + key: Helpers.command(), + min_version: non_neg_integer(), + max_version: non_neg_integer() + } + defstruct [:key, :min_version, :max_version] + end + + def new!(_opts \\ []) do + %__MODULE__{ + commands: [ + %Command{key: :publish, min_version: 1, max_version: 2}, + %Command{key: :deliver, min_version: 1, max_version: 2}, + %Command{key: :declare_producer, min_version: 1, max_version: 1}, + %Command{key: :publish_confirm, min_version: 1, max_version: 1}, + %Command{key: :publish_error, min_version: 1, max_version: 1}, + %Command{key: :query_producer_sequence, min_version: 1, max_version: 1}, + %Command{key: :delete_producer, min_version: 1, max_version: 1}, + %Command{key: :subscribe, min_version: 1, max_version: 1}, + %Command{key: :credit, min_version: 1, max_version: 1}, + %Command{key: :store_offset, min_version: 1, max_version: 1}, + %Command{key: :query_offset, min_version: 1, max_version: 1}, + %Command{key: :unsubscribe, min_version: 1, max_version: 1}, + %Command{key: :create_stream, min_version: 1, max_version: 1}, + %Command{key: :delete_stream, min_version: 1, max_version: 1}, + %Command{key: :query_metadata, min_version: 1, max_version: 1}, + %Command{key: :metadata_update, min_version: 1, max_version: 1}, + %Command{key: :peer_properties, min_version: 1, max_version: 1}, + %Command{key: :sasl_handshake, min_version: 1, max_version: 1}, + %Command{key: :sasl_authenticate, min_version: 1, max_version: 1}, + %Command{key: :tune, min_version: 1, max_version: 1}, + %Command{key: :open, min_version: 1, max_version: 1}, + %Command{key: :close, min_version: 1, max_version: 1}, + %Command{key: :heartbeat, min_version: 1, max_version: 1}, + %Command{key: :route, min_version: 1, max_version: 1}, + %Command{key: :partitions, min_version: 1, max_version: 1}, + %Command{key: :consumer_update, min_version: 1, max_version: 1}, + %Command{key: :exchange_command_versions, min_version: 1, max_version: 1}, + %Command{key: :stream_stats, min_version: 1, max_version: 1}, + %Command{key: :create_super_stream, min_version: 1, max_version: 1}, + %Command{key: :delete_super_stream, min_version: 1, max_version: 1} + ] + } + end + end + + defmodule StreamStatsRequestData do + @moduledoc false + @enforce_keys [:stream_name] + @type t :: %{stream_name: String.t()} + defstruct [:stream_name] + end + + defmodule StreamStatsResponseData do + @enforce_keys [:stats] + @type t :: %{stats: %{String.t() => integer()}} + defstruct [:stats] + end + + defmodule CreateSuperStreamRequestData do + @enforce_keys [:name, :partitions, :arguments] + @type t :: %{ + name: String.t(), + partitions: [{String.t(), String.t()}], + arguments: Keyword.t(String.t()) + } + defstruct [:name, :partitions, :arguments] + end + + defmodule CreateSuperStreamResponseData do + @moduledoc false + @type t :: %{} + defstruct [] + end + + defmodule DeleteSuperStreamRequestData do + @moduledoc false + @enforce_keys [:name] + @type t :: %{name: String.t()} + defstruct [:name] + end + + defmodule DeleteSuperStreamResponseData do + @moduledoc false + @type t :: %{} + defstruct [] + end +end diff --git a/lib/message/decoder.ex b/lib/message/decoder.ex index 0df3551..3695d10 100644 --- a/lib/message/decoder.ex +++ b/lib/message/decoder.ex @@ -1,210 +1,73 @@ defmodule RabbitMQStream.Message.Decoder do @moduledoc false + import RabbitMQStream.Message.Helpers + alias RabbitMQStream.Message.Data - alias RabbitMQStream.Message.{Response, Request, Frame} + alias RabbitMQStream.Message.{Response, Request} + import Bitwise - alias RabbitMQStream.Message + def decode(buffer) do + <> = buffer - alias RabbitMQStream.Message.Data.{ - TuneData, - HeartbeatData, - CloseData, - MetadataUpdateData, - QueryMetadataData, - BrokerData, - StreamData, - PublishConfirmData, - PublishErrorData, - DeliverData - } + command = decode_command(key) - def parse(<>) do - <> = <<0b0::1, key::bits>> - command = Frame.code_to_command(key) - - case flag do - 0b1 -> - %Response{version: version, command: command} - - 0b0 -> - %Request{version: version, command: command} + if (key &&& 0b1000_0000_0000_0000) > 0 do + %Response{version: version, command: command} + else + %Request{version: version, command: command} end - |> parse(buffer) + |> decode(buffer) end - def parse(%Response{command: command} = response, buffer) + def decode(%Response{command: command} = response, buffer) when command in [ :close, :create_stream, :delete_stream, - :declare_publisher, - :delete_publisher, + :declare_producer, + :delete_producer, :subscribe, :unsubscribe, :credit, :query_offset, - :query_publisher_sequence, + :query_producer_sequence, :peer_properties, :sasl_handshake, :sasl_authenticate, - :tune, - :open + :open, + :route, + :partitions, + :exchange_command_versions, + :consumer_update, + :stream_stats, + :create_super_stream, + :delete_super_stream ] do <> = buffer - %{ - response - | data: Message.Data.decode_data(command, buffer), - correlation_id: correlation_id, - code: Frame.response_code_to_atom(code) - } - end - - def parse(%Request{command: :close} = request, buffer) do - <> = buffer - - <> = buffer + response = %{response | correlation_id: correlation_id, code: decode_code(code)} - {"", reason} = Message.Data.fetch_string(buffer) - - data = %CloseData{code: code, reason: reason} - - %{request | data: data, correlation_id: correlation_id} + %{response | data: Data.decode(response, buffer)} end - def parse(%Request{command: :tune} = request, buffer) do - <> = buffer - - data = %TuneData{frame_max: frame_max, heartbeat: heartbeat} - - %{request | data: data} - end - - def parse(%Request{command: :heartbeat} = request, "") do - %{request | data: %HeartbeatData{}} - end - - def parse(%Request{command: :metadata_update} = request, buffer) do - <> = buffer - - {"", stream_name} = Message.Data.fetch_string(buffer) - - data = %MetadataUpdateData{stream_name: stream_name} - - %{request | data: data, code: Frame.response_code_to_atom(code)} - end - - def parse(%Response{command: :query_metadata} = response, buffer) do + def decode(%{command: command} = response, buffer) + when command in [:close, :query_metadata, :consumer_update] do <> = buffer - {buffer, brokers} = - Message.Data.decode_array(buffer, fn buffer, acc -> - <> = buffer - - <> = buffer - - <> = buffer - - data = %BrokerData{ - reference: reference, - host: host, - port: port - } - - {buffer, [data] ++ acc} - end) - - {"", streams} = - Message.Data.decode_array(buffer, fn buffer, acc -> - << - size::integer-size(16), - name::binary-size(size), - code::unsigned-integer-size(16), - leader::unsigned-integer-size(16), - buffer::binary - >> = buffer - - {buffer, replicas} = - Message.Data.decode_array(buffer, fn buffer, acc -> - <> = buffer - - {buffer, [replica] ++ acc} - end) - - data = %StreamData{ - code: code, - name: name, - leader: leader, - replicas: replicas - } - - {buffer, [data] ++ acc} - end) - - data = %QueryMetadataData{brokers: brokers, streams: streams} - - %{response | correlation_id: correlation_id, data: data} - end - - def parse(%Request{command: :publish_confirm} = request, buffer) do - <> = buffer - - {"", publishing_ids} = - Message.Data.decode_array(buffer, fn buffer, acc -> - <> = buffer - {buffer, [publishing_id] ++ acc} - end) - - data = %PublishConfirmData{publisher_id: publisher_id, publishing_ids: publishing_ids} - - %{request | data: data} + response = %{response | correlation_id: correlation_id} + %{response | data: Data.decode(response, buffer)} end - def parse(%Request{command: :publish_error} = request, buffer) do - <> = buffer - - {"", errors} = - Message.Data.decode_array(buffer, fn buffer, acc -> - << - publishing_id::unsigned-integer-size(64), - code::unsigned-integer-size(16), - buffer::binary - >> = buffer - - entry = %PublishErrorData.Error{ - code: Frame.response_code_to_atom(code), - publishing_id: publishing_id - } - - {buffer, [entry] ++ acc} - end) - - data = %PublishErrorData{publisher_id: publisher_id, errors: errors} - - %{request | data: data} - end - - def parse(%Request{version: 1, command: :deliver} = request, buffer) do - <> = buffer - - osiris_chunk = RabbitMQStream.OsirisChunk.decode!(rest) - - data = %DeliverData{subscription_id: subscription_id, osiris_chunk: osiris_chunk} - - %{request | data: data} - end - - def parse(%Request{version: 2, command: :deliver} = request, buffer) do - <> = buffer - - osiris_chunk = RabbitMQStream.OsirisChunk.decode!(rest) - - data = %DeliverData{ - subscription_id: subscription_id, - committed_offset: committed_offset, - osiris_chunk: osiris_chunk - } - - %{request | data: data} + def decode(%{command: command} = action, buffer) + when command in [ + :tune, + :heartbeat, + :metadata_update, + :publish_confirm, + :publish_error, + :deliver, + :store_offset + ] do + %{action | data: Data.decode(action, buffer)} end end diff --git a/lib/message/encoder.ex b/lib/message/encoder.ex index 8fbe129..dde2d97 100644 --- a/lib/message/encoder.ex +++ b/lib/message/encoder.ex @@ -1,178 +1,15 @@ defmodule RabbitMQStream.Message.Encoder do @moduledoc false + import RabbitMQStream.Message.Helpers - alias RabbitMQStream.Message.{Response, Request, Frame} + alias RabbitMQStream.Message.Response + alias RabbitMQStream.Message.Request + alias RabbitMQStream.Message.Data - def encode!(command) do - payload = encode_payload!(command) + def encode(command) do + header = bake_header(command) - wrap(command, payload) - end - - defp encode_payload!(%Request{command: :peer_properties, data: data}) do - properties = encode_map(data.peer_properties) - - <> - end - - defp encode_payload!(%Request{command: :sasl_handshake}) do - <<>> - end - - defp encode_payload!(%Request{command: :sasl_authenticate, data: data}) do - mechanism = encode_string(data.mechanism) - - credentials = - encode_bytes("\u0000#{data.sasl_opaque_data[:username]}\u0000#{data.sasl_opaque_data[:password]}") - - <> - end - - defp encode_payload!(%Request{command: :open, data: data}) do - vhost = encode_string(data.vhost) - - <> - end - - defp encode_payload!(%Request{command: :heartbeat}) do - <<>> - end - - defp encode_payload!(%Request{command: :tune, data: data}) do - <> - end - - defp encode_payload!(%Request{command: :close, data: data}) do - reason = encode_string(data.reason) - - <> - end - - defp encode_payload!(%Request{command: :create_stream, data: data}) do - stream_name = encode_string(data.stream_name) - arguments = encode_map(data.arguments) - - <> - end - - defp encode_payload!(%Request{command: :delete_stream, data: data}) do - stream_name = encode_string(data.stream_name) - - <> - end - - defp encode_payload!(%Request{command: :store_offset, data: data}) do - offset_reference = encode_string(data.offset_reference) - stream_name = encode_string(data.stream_name) - - << - offset_reference::binary, - stream_name::binary, - data.offset::unsigned-integer-size(64) - >> - end - - defp encode_payload!(%Request{command: :query_offset, data: data}) do - offset_reference = encode_string(data.offset_reference) - stream_name = encode_string(data.stream_name) - - << - offset_reference::binary, - stream_name::binary - >> - end - - defp encode_payload!(%Request{command: :declare_publisher, data: data}) do - publisher_reference = encode_string(data.publisher_reference) - stream_name = encode_string(data.stream_name) - - << - data.id::unsigned-integer-size(8), - publisher_reference::binary, - stream_name::binary - >> - end - - defp encode_payload!(%Request{command: :delete_publisher, data: data}) do - <> - end - - defp encode_payload!(%Request{command: :query_metadata, data: data}) do - streams = - data.streams - |> Enum.map(&encode_string/1) - |> encode_array() - - <> - end - - defp encode_payload!(%Request{command: :query_publisher_sequence, data: data}) do - publisher_reference = encode_string(data.publisher_reference) - stream_name = encode_string(data.stream_name) - - <> - end - - defp encode_payload!(%Request{command: :publish, data: data}) do - messages = - encode_array( - for {publishing_id, message} <- data.published_messages do - <> - end - ) - - <> - end - - defp encode_payload!(%Request{command: :subscribe, data: data}) do - stream_name = encode_string(data.stream_name) - - offset = - case data.offset do - :first -> <<1::unsigned-integer-size(16)>> - :last -> <<2::unsigned-integer-size(16)>> - :next -> <<3::unsigned-integer-size(16)>> - {:offset, offset} -> <<4::unsigned-integer-size(16), offset::unsigned-integer-size(64)>> - {:timestamp, timestamp} -> <<5::unsigned-integer-size(16), timestamp::integer-size(64)>> - end - - properties = encode_map(data.properties) - - << - data.subscription_id::unsigned-integer-size(8), - stream_name::binary, - offset::binary, - data.credit::unsigned-integer-size(16), - properties::binary - >> - end - - defp encode_payload!(%Request{command: :unsubscribe, data: data}) do - <> - end - - defp encode_payload!(%Request{command: :credit, data: data}) do - << - data.subscription_id::unsigned-integer-size(8), - data.credit::unsigned-integer-size(16) - >> - end - - defp encode_payload!(%Response{command: :tune, data: data}) do - << - data.frame_max::unsigned-integer-size(32), - data.heartbeat::unsigned-integer-size(32) - >> - end - - defp encode_payload!(%Response{command: :close, code: code}) do - << - Frame.atom_to_response_code(code)::unsigned-integer-size(16) - >> - end - - defp wrap(request, payload) do - header = bake_header(request) + payload = Data.encode(command) buffer = <> @@ -190,15 +27,21 @@ defmodule RabbitMQStream.Message.Encoder do :create_stream, :delete_stream, :query_offset, - :declare_publisher, - :delete_publisher, + :declare_producer, + :delete_producer, :query_metadata, - :query_publisher_sequence, + :query_producer_sequence, :subscribe, - :unsubscribe + :unsubscribe, + :route, + :partitions, + :exchange_command_versions, + :stream_stats, + :create_super_stream, + :delete_super_stream ] do << - Frame.command_to_code(command)::unsigned-integer-size(16), + encode_command(command)::unsigned-integer-size(16), version::unsigned-integer-size(16), correlation_id::unsigned-integer-size(32) >> @@ -206,58 +49,22 @@ defmodule RabbitMQStream.Message.Encoder do defp bake_header(%Request{command: command, version: version}) when command in [:heartbeat, :store_offset, :publish, :credit] do - <> + <> end - defp bake_header(%Response{command: command, version: version, correlation_id: correlation_id}) - when command in [:close] do + defp bake_header(%Response{command: command, version: version, correlation_id: correlation_id, code: code}) + when command in [:close, :consumer_update] do << 0b1::1, - Frame.command_to_code(command)::unsigned-integer-size(15), + encode_command(command)::unsigned-integer-size(15), version::unsigned-integer-size(16), - correlation_id::unsigned-integer-size(32) + correlation_id::unsigned-integer-size(32), + encode_code(code)::unsigned-integer-size(16) >> end defp bake_header(%Response{command: command, version: version}) when command in [:tune] do - <<0b1::1, Frame.command_to_code(command)::unsigned-integer-size(15), version::unsigned-integer-size(16)>> - end - - defp encode_string(value) when is_atom(value) do - encode_string(Atom.to_string(value)) - end - - defp encode_string(nil) do - <<-1::integer-size(16)>> - end - - defp encode_string(str) do - <> - end - - defp encode_bytes(bytes) do - <> - end - - defp encode_array([]) do - <<0::integer-size(32)>> - end - - defp encode_array(arr) do - size = Enum.count(arr) - arr = arr |> Enum.reduce(&<>/2) - - <> - end - - defp encode_map(nil) do - encode_array([]) - end - - defp encode_map(list) do - list - |> Enum.map(fn {key, value} -> encode_string(key) <> encode_string(value) end) - |> encode_array() + <<0b1::1, encode_command(command)::unsigned-integer-size(15), version::unsigned-integer-size(16)>> end end diff --git a/lib/message/frame.ex b/lib/message/frame.ex deleted file mode 100644 index d5ae805..0000000 --- a/lib/message/frame.ex +++ /dev/null @@ -1,78 +0,0 @@ -defmodule RabbitMQStream.Message.Frame do - @moduledoc false - - @commands %{ - 0x0001 => :declare_publisher, - 0x0002 => :publish, - 0x0003 => :publish_confirm, - 0x0004 => :publish_error, - 0x0005 => :query_publisher_sequence, - 0x0006 => :delete_publisher, - 0x0007 => :subscribe, - 0x0008 => :deliver, - 0x0009 => :credit, - 0x000A => :store_offset, - 0x000B => :query_offset, - 0x000C => :unsubscribe, - 0x000D => :create_stream, - 0x000E => :delete_stream, - 0x000F => :query_metadata, - 0x0010 => :metadata_update, - 0x0011 => :peer_properties, - 0x0012 => :sasl_handshake, - 0x0013 => :sasl_authenticate, - 0x0014 => :tune, - 0x0015 => :open, - 0x0016 => :close, - 0x0017 => :heartbeat, - 0x0018 => :route, - 0x0019 => :partitions, - 0x001A => :consumer_update, - 0x001B => :exchange_command_versions, - 0x001C => :stream_stats, - 0x001D => :create_super_stream, - 0x001E => :delete_super_stream - } - - @codes Enum.into(@commands, %{}, fn {code, command} -> {command, code} end) - - def command_to_code(command) do - @codes[command] - end - - def code_to_command(code) do - @commands[code] - end - - @response_codes %{ - 0x01 => :ok, - 0x02 => :stream_does_not_exist, - 0x03 => :subscription_id_already_exists, - 0x04 => :subscription_id_does_not_exist, - 0x05 => :stream_already_exists, - 0x06 => :stream_not_available, - 0x07 => :sasl_mechanism_not_supported, - 0x08 => :authentication_failure, - 0x09 => :sasl_error, - 0x0A => :sasl_challenge, - 0x0B => :sasl_authentication_failure_loopback, - 0x0C => :virtual_host_access_failure, - 0x0D => :unknown_frame, - 0x0E => :frame_too_large, - 0x0F => :internal_error, - 0x10 => :access_refused, - 0x11 => :precondition_failed, - 0x12 => :publisher_does_not_exist, - 0x13 => :no_offset - } - - @code_responses Enum.into(@response_codes, %{}, fn {code, command} -> {command, code} end) - - def response_code_to_atom(code) do - @response_codes[code] - end - - def atom_to_response_code(atom) do - @code_responses[atom] - end -end diff --git a/lib/message/helpers.ex b/lib/message/helpers.ex new file mode 100644 index 0000000..94874e1 --- /dev/null +++ b/lib/message/helpers.ex @@ -0,0 +1,215 @@ +defmodule RabbitMQStream.Message.Helpers do + @type command :: + :declare_producer + | :publish + | :publish_confirm + | :publish_error + | :query_producer_sequence + | :delete_producer + | :subscribe + | :deliver + | :credit + | :store_offset + | :query_offset + | :unsubscribe + | :create_stream + | :delete_stream + | :query_metadata + | :metadata_update + | :peer_properties + | :sasl_handshake + | :sasl_authenticate + | :tune + | :open + | :close + | :heartbeat + | :route + | :partitions + | :consumer_update + | :exchange_command_versions + | :stream_stats + | :create_super_stream + | :delete_super_stream + + @type code :: + :ok + | :stream_does_not_exist + | :subscription_id_already_exists + | :subscription_id_does_not_exist + | :stream_already_exists + | :stream_not_available + | :sasl_mechanism_not_supported + | :authentication_failure + | :sasl_error + | :sasl_challenge + | :sasl_authentication_failure_loopback + | :virtual_host_access_failure + | :unknown_frame + | :frame_too_large + | :internal_error + | :access_refused + | :precondition_failed + | :producer_does_not_exist + | :no_offset + + @commands %{ + 0x0001 => :declare_producer, + 0x0002 => :publish, + 0x0003 => :publish_confirm, + 0x0004 => :publish_error, + 0x0005 => :query_producer_sequence, + 0x0006 => :delete_producer, + 0x0007 => :subscribe, + 0x0008 => :deliver, + 0x0009 => :credit, + 0x000A => :store_offset, + 0x000B => :query_offset, + 0x000C => :unsubscribe, + 0x000D => :create_stream, + 0x000E => :delete_stream, + 0x000F => :query_metadata, + 0x0010 => :metadata_update, + 0x0011 => :peer_properties, + 0x0012 => :sasl_handshake, + 0x0013 => :sasl_authenticate, + 0x0014 => :tune, + 0x0015 => :open, + 0x0016 => :close, + 0x0017 => :heartbeat, + 0x0018 => :route, + 0x0019 => :partitions, + 0x001A => :consumer_update, + 0x001B => :exchange_command_versions, + 0x001C => :stream_stats, + 0x001D => :create_super_stream, + 0x001E => :delete_super_stream + } + + @codes Enum.into(@commands, %{}, fn {code, command} -> {command, code} end) + + def encode_command(command) do + @codes[command] + end + + def decode_command(key) do + @commands[Bitwise.band(key, 0b0111_1111_1111_1111)] + end + + @response_codes %{ + 0x01 => :ok, + 0x02 => :stream_does_not_exist, + 0x03 => :subscription_id_already_exists, + 0x04 => :subscription_id_does_not_exist, + 0x05 => :stream_already_exists, + 0x06 => :stream_not_available, + 0x07 => :sasl_mechanism_not_supported, + 0x08 => :authentication_failure, + 0x09 => :sasl_error, + 0x0A => :sasl_challenge, + 0x0B => :sasl_authentication_failure_loopback, + 0x0C => :virtual_host_access_failure, + 0x0D => :unknown_frame, + 0x0E => :frame_too_large, + 0x0F => :internal_error, + 0x10 => :access_refused, + 0x11 => :precondition_failed, + 0x12 => :producer_does_not_exist, + 0x13 => :no_offset + } + + @code_responses Enum.into(@response_codes, %{}, fn {code, command} -> {command, code} end) + + def decode_code(code) do + @response_codes[code] + end + + def encode_code(atom) do + @code_responses[atom] + end + + def encode_string(value) when is_atom(value) do + encode_string(Atom.to_string(value)) + end + + def encode_string(value) when is_integer(value) do + encode_string(Integer.to_string(value)) + end + + def encode_string(nil) do + <<-1::integer-size(16)>> + end + + def encode_string(str) do + <> + end + + def encode_bytes(bytes) do + <> + end + + def encode_array([]) do + <<0::integer-size(32)>> + end + + def encode_array(arr) do + size = Enum.count(arr) + arr = arr |> Enum.reduce("", &<>/2) + + <> + end + + def encode_array(arr, foo) when is_function(foo, 1) do + size = Enum.count(arr) + arr = arr |> Enum.map(foo) |> Enum.reduce("", &<>/2) + + <> + end + + def encode_map(nil) do + encode_array([]) + end + + def encode_map(list) do + list + |> Enum.map(fn {key, value} -> encode_string(key) <> encode_string(value) end) + |> encode_array() + end + + def decode_string(<>) do + {rest, to_string(text)} + end + + def decode_array("", _) do + {"", []} + end + + def decode_array(<<0::integer-size(32), buffer::binary>>, _) do + {buffer, []} + end + + def decode_array(<>, foo) do + Enum.reduce(0..(size - 1), {buffer, []}, fn _, {buffer, acc} -> + foo.(buffer, acc) + end) + end + + def encode_offset(offset) do + case offset do + :first -> <<1::unsigned-integer-size(16)>> + :last -> <<2::unsigned-integer-size(16)>> + :next -> <<3::unsigned-integer-size(16)>> + {:offset, offset} -> <<4::unsigned-integer-size(16), offset::unsigned-integer-size(64)>> + {:timestamp, timestamp} -> <<5::unsigned-integer-size(16), timestamp::integer-size(64)>> + end + end + + def decode_offset(buffer) do + case buffer do + <<1::unsigned-integer-size(16), rest::binary>> -> {rest, :first} + <<2::unsigned-integer-size(16), rest::binary>> -> {rest, :last} + <<3::unsigned-integer-size(16), rest::binary>> -> {rest, :next} + <<4::unsigned-integer-size(16), offset::unsigned-integer-size(64), rest::binary>> -> {rest, {:offset, offset}} + <<5::unsigned-integer-size(16), timestamp::integer-size(64), rest::binary>> -> {rest, {:timestamp, timestamp}} + end + end +end diff --git a/lib/message/message.ex b/lib/message/message.ex new file mode 100644 index 0000000..3c89a6a --- /dev/null +++ b/lib/message/message.ex @@ -0,0 +1,391 @@ +defmodule RabbitMQStream.Message do + @moduledoc false + require Logger + + alias RabbitMQStream.Connection + + alias RabbitMQStream.Message.Types + + defmodule Request do + @type t :: %__MODULE__{ + version: non_neg_integer, + correlation_id: non_neg_integer, + command: atom, + data: term(), + code: non_neg_integer + } + + @enforce_keys [:command, :version] + + defstruct [ + :version, + :correlation_id, + :command, + :data, + :code + ] + end + + defmodule Response do + @type t :: %__MODULE__{ + version: non_neg_integer, + correlation_id: non_neg_integer, + command: atom, + data: Types.t(), + code: non_neg_integer + } + + @enforce_keys [:command, :version] + defstruct [ + :version, + :command, + :correlation_id, + :data, + :code + ] + end + + @version Mix.Project.config()[:version] + + def new_request(%Connection{} = conn, :peer_properties, _) do + %Request{ + version: 1, + correlation_id: conn.correlation_sequence, + command: :peer_properties, + data: %Types.PeerPropertiesData{ + peer_properties: [ + {"product", "RabbitMQ Stream Client"}, + {"information", "Development"}, + {"version", @version}, + {"platform", "Elixir"} + ] + } + } + end + + def new_request(%Connection{} = conn, :sasl_handshake, _) do + %Request{ + version: 1, + correlation_id: conn.correlation_sequence, + command: :sasl_handshake, + data: %Types.SaslHandshakeData{} + } + end + + def new_request(%Connection{} = conn, :sasl_authenticate, _) do + cond do + Enum.member?(conn.mechanisms, "PLAIN") -> + %Request{ + version: 1, + correlation_id: conn.correlation_sequence, + command: :sasl_authenticate, + data: %Types.SaslAuthenticateData{ + mechanism: "PLAIN", + sasl_opaque_data: [ + username: conn.options[:username], + password: conn.options[:password] + ] + } + } + + true -> + raise "Unsupported SASL mechanism: #{conn.mechanisms}" + end + end + + def new_request(%Connection{} = conn, :tune, _) do + %Request{ + version: 1, + correlation_id: conn.correlation_sequence, + command: :tune, + data: %Types.TuneData{ + frame_max: conn.options[:frame_max], + heartbeat: conn.options[:heartbeat] + } + } + end + + def new_request(%Connection{} = conn, :open, _) do + %Request{ + version: 1, + correlation_id: conn.correlation_sequence, + command: :open, + data: %Types.OpenRequestData{ + vhost: conn.options[:vhost] + } + } + end + + def new_request(%Connection{}, :heartbeat, _) do + %Request{ + version: 1, + command: :heartbeat, + data: %Types.HeartbeatData{} + } + end + + def new_request(%Connection{} = conn, :close, opts) do + %Request{ + version: 1, + command: :close, + correlation_id: conn.correlation_sequence, + data: %Types.CloseRequestData{ + code: opts[:code], + reason: opts[:reason] + } + } + end + + def new_request(%Connection{} = conn, :create_stream, opts) do + %Request{ + version: 1, + command: :create_stream, + correlation_id: conn.correlation_sequence, + data: %Types.CreateStreamRequestData{ + stream_name: opts[:name], + arguments: opts[:arguments] || [] + } + } + end + + def new_request(%Connection{} = conn, :delete_stream, opts) do + %Request{ + version: 1, + command: :delete_stream, + correlation_id: conn.correlation_sequence, + data: %Types.DeleteStreamRequestData{ + stream_name: opts[:name] + } + } + end + + def new_request(%Connection{} = conn, :store_offset, opts) do + %Request{ + version: 1, + command: :store_offset, + correlation_id: conn.correlation_sequence, + data: %Types.StoreOffsetRequestData{ + stream_name: opts[:stream_name], + offset_reference: opts[:offset_reference], + offset: opts[:offset] + } + } + end + + def new_request(%Connection{} = conn, :query_offset, opts) do + %Request{ + version: 1, + command: :query_offset, + correlation_id: conn.correlation_sequence, + data: %Types.QueryOffsetRequestData{ + stream_name: opts[:stream_name], + offset_reference: opts[:offset_reference] + } + } + end + + def new_request(%Connection{} = conn, :declare_producer, opts) do + %Request{ + version: 1, + command: :declare_producer, + correlation_id: conn.correlation_sequence, + data: %Types.DeclareProducerRequestData{ + id: opts[:id], + producer_reference: opts[:producer_reference], + stream_name: opts[:stream_name] + } + } + end + + def new_request(%Connection{} = conn, :delete_producer, opts) do + %Request{ + version: 1, + command: :delete_producer, + correlation_id: conn.correlation_sequence, + data: %Types.DeleteProducerRequestData{ + producer_id: opts[:producer_id] + } + } + end + + def new_request(%Connection{} = conn, :query_metadata, opts) do + %Request{ + version: 1, + command: :query_metadata, + correlation_id: conn.correlation_sequence, + data: %Types.QueryMetadataRequestData{ + streams: opts[:streams] + } + } + end + + def new_request(%Connection{} = conn, :query_producer_sequence, opts) do + %Request{ + version: 1, + command: :query_producer_sequence, + correlation_id: conn.correlation_sequence, + data: %Types.QueryProducerSequenceRequestData{ + stream_name: opts[:stream_name], + producer_reference: opts[:producer_reference] + } + } + end + + def new_request(%Connection{} = conn, :publish, opts) when conn.commands.publish.max >= 2 do + %Request{ + version: 2, + command: :publish, + data: %Types.PublishData{ + producer_id: opts[:producer_id], + messages: opts[:messages] + } + } + end + + def new_request(%Connection{}, :publish, opts) do + %Request{ + version: 1, + command: :publish, + data: %Types.PublishData{ + producer_id: opts[:producer_id], + messages: opts[:messages] + } + } + end + + def new_request(%Connection{} = conn, :subscribe, opts) do + %Request{ + version: 1, + command: :subscribe, + correlation_id: conn.correlation_sequence, + data: Types.SubscribeRequestData.new!(opts) + } + end + + def new_request(%Connection{} = conn, :unsubscribe, opts) do + %Request{ + version: 1, + command: :unsubscribe, + correlation_id: conn.correlation_sequence, + data: %Types.UnsubscribeRequestData{ + subscription_id: opts[:subscription_id] + } + } + end + + def new_request(%Connection{} = conn, :credit, opts) do + %Request{ + version: 1, + command: :credit, + correlation_id: conn.correlation_sequence, + data: %Types.CreditRequestData{ + credit: opts[:credit], + subscription_id: opts[:subscription_id] + } + } + end + + def new_request(%Connection{} = conn, :route, opts) do + %Request{ + version: 1, + command: :route, + correlation_id: conn.correlation_sequence, + data: %Types.RouteRequestData{ + super_stream: opts[:super_stream], + routing_key: opts[:routing_key] + } + } + end + + def new_request(%Connection{} = conn, :partitions, opts) do + %Request{ + version: 1, + command: :partitions, + correlation_id: conn.correlation_sequence, + data: %Types.PartitionsQueryRequestData{ + super_stream: opts[:super_stream] + } + } + end + + def new_request(%Connection{} = conn, :exchange_command_versions, opts) do + %Request{ + version: 1, + command: :exchange_command_versions, + correlation_id: conn.correlation_sequence, + data: Types.ExchangeCommandVersionsData.new!(opts) + } + end + + def new_request(%Connection{} = conn, :stream_stats, opts) do + %Request{ + version: 1, + command: :stream_stats, + correlation_id: conn.correlation_sequence, + data: %Types.StreamStatsRequestData{ + stream_name: opts[:stream_name] + } + } + end + + def new_request(%Connection{} = conn, :create_super_stream, opts) do + %Request{ + version: 1, + command: :create_super_stream, + correlation_id: conn.correlation_sequence, + data: struct(Types.CreateSuperStreamRequestData, opts) + } + end + + def new_request(%Connection{} = conn, :delete_super_stream, opts) do + %Request{ + version: 1, + command: :delete_super_stream, + correlation_id: conn.correlation_sequence, + data: struct(Types.DeleteSuperStreamRequestData, opts) + } + end + + def new_response(%Connection{options: options}, :tune, correlation_id: correlation_id) do + %Response{ + version: 1, + command: :tune, + correlation_id: correlation_id, + data: %Types.TuneData{ + frame_max: options[:frame_max], + heartbeat: options[:heartbeat] + } + } + end + + def new_response(%Connection{}, :heartbeat, correlation_id: correlation_id) do + %Response{ + version: 1, + command: :heartbeat, + correlation_id: correlation_id, + data: %Types.HeartbeatData{} + } + end + + def new_response(%Connection{}, :close, correlation_id: correlation_id, code: code) do + %Response{ + version: 1, + correlation_id: correlation_id, + command: :close, + data: %Types.CloseResponseData{}, + code: code + } + end + + def new_response(%Connection{}, :consumer_update, opts) do + %Response{ + version: 1, + correlation_id: opts[:correlation_id], + command: :consumer_update, + data: %Types.ConsumerUpdateResponseData{ + offset: opts[:offset] + }, + code: opts[:code] + } + end +end diff --git a/lib/message/request.ex b/lib/message/request.ex deleted file mode 100644 index 47e0d60..0000000 --- a/lib/message/request.ex +++ /dev/null @@ -1,272 +0,0 @@ -defmodule RabbitMQStream.Message.Request do - @moduledoc false - require Logger - alias __MODULE__ - - alias RabbitMQStream.Connection - - alias RabbitMQStream.Message.Data.{ - TuneData, - OpenData, - PeerPropertiesData, - SaslAuthenticateData, - SaslHandshakeData, - HeartbeatData, - CloseData, - CreateStreamData, - DeleteStreamData, - StoreOffsetData, - QueryOffsetData, - DeclarePublisherData, - DeletePublisherData, - QueryMetadataData, - QueryPublisherSequenceData, - PublishData, - SubscribeRequestData, - UnsubscribeRequestData, - CreditRequestData - } - - defstruct [ - :version, - :correlation_id, - :command, - :data, - :code - ] - - @version Mix.Project.config()[:version] - - def new!(%Connection{} = conn, :peer_properties, _) do - %Request{ - version: conn.version, - correlation_id: conn.correlation_sequence, - command: :peer_properties, - data: %PeerPropertiesData{ - peer_properties: [ - {"product", "RabbitMQ Stream Client"}, - {"information", "Development"}, - {"version", @version}, - {"platform", "Elixir"} - ] - } - } - end - - def new!(%Connection{} = conn, :sasl_handshake, _) do - %Request{ - version: conn.version, - correlation_id: conn.correlation_sequence, - command: :sasl_handshake, - data: %SaslHandshakeData{} - } - end - - def new!(%Connection{} = conn, :sasl_authenticate, _) do - cond do - Enum.member?(conn.mechanisms, "PLAIN") -> - %Request{ - version: conn.version, - correlation_id: conn.correlation_sequence, - command: :sasl_authenticate, - data: %SaslAuthenticateData{ - mechanism: "PLAIN", - sasl_opaque_data: [ - username: conn.options[:username], - password: conn.options[:password] - ] - } - } - - true -> - raise "Unsupported SASL mechanism: #{conn.mechanisms}" - end - end - - def new!(%Connection{} = conn, :tune, _) do - %Request{ - version: conn.version, - correlation_id: conn.correlation_sequence, - command: :tune, - data: %TuneData{ - frame_max: conn.options[:frame_max], - heartbeat: conn.options[:heartbeat] - } - } - end - - def new!(%Connection{} = conn, :open, _) do - %Request{ - version: conn.version, - correlation_id: conn.correlation_sequence, - command: :open, - data: %OpenData{ - vhost: conn.options[:vhost] - } - } - end - - def new!(%Connection{} = conn, :heartbeat, _) do - %Request{ - version: conn.version, - command: :heartbeat, - data: %HeartbeatData{} - } - end - - def new!(%Connection{} = conn, :close, opts) do - %Request{ - version: conn.version, - command: :close, - correlation_id: conn.correlation_sequence, - data: %CloseData{ - code: opts[:code], - reason: opts[:reason] - } - } - end - - def new!(%Connection{} = conn, :create_stream, opts) do - %Request{ - version: conn.version, - command: :create_stream, - correlation_id: conn.correlation_sequence, - data: %CreateStreamData{ - stream_name: opts[:name], - arguments: opts[:arguments] || [] - } - } - end - - def new!(%Connection{} = conn, :delete_stream, opts) do - %Request{ - version: conn.version, - command: :delete_stream, - correlation_id: conn.correlation_sequence, - data: %DeleteStreamData{ - stream_name: opts[:name] - } - } - end - - def new!(%Connection{} = conn, :store_offset, opts) do - %Request{ - version: conn.version, - command: :store_offset, - correlation_id: conn.correlation_sequence, - data: %StoreOffsetData{ - stream_name: opts[:stream_name], - offset_reference: opts[:offset_reference], - offset: opts[:offset] - } - } - end - - def new!(%Connection{} = conn, :query_offset, opts) do - %Request{ - version: conn.version, - command: :query_offset, - correlation_id: conn.correlation_sequence, - data: %QueryOffsetData{ - stream_name: opts[:stream_name], - offset_reference: opts[:offset_reference] - } - } - end - - def new!(%Connection{} = conn, :declare_publisher, opts) do - %Request{ - version: conn.version, - command: :declare_publisher, - correlation_id: conn.correlation_sequence, - data: %DeclarePublisherData{ - id: conn.publisher_sequence, - publisher_reference: opts[:publisher_reference], - stream_name: opts[:stream_name] - } - } - end - - def new!(%Connection{} = conn, :delete_publisher, opts) do - %Request{ - version: conn.version, - command: :delete_publisher, - correlation_id: conn.correlation_sequence, - data: %DeletePublisherData{ - publisher_id: opts[:publisher_id] - } - } - end - - def new!(%Connection{} = conn, :query_metadata, opts) do - %Request{ - version: conn.version, - command: :query_metadata, - correlation_id: conn.correlation_sequence, - data: %QueryMetadataData{ - streams: opts[:streams] - } - } - end - - def new!(%Connection{} = conn, :query_publisher_sequence, opts) do - %Request{ - version: conn.version, - command: :query_publisher_sequence, - correlation_id: conn.correlation_sequence, - data: %QueryPublisherSequenceData{ - stream_name: opts[:stream_name], - publisher_reference: opts[:publisher_reference] - } - } - end - - def new!(%Connection{} = conn, :publish, opts) do - %Request{ - version: conn.version, - command: :publish, - data: %PublishData{ - publisher_id: opts[:publisher_id], - published_messages: opts[:published_messages] - } - } - end - - def new!(%Connection{} = conn, :subscribe, opts) do - %Request{ - version: conn.version, - command: :subscribe, - correlation_id: conn.correlation_sequence, - data: %SubscribeRequestData{ - credit: opts[:credit], - offset: opts[:offset], - properties: opts[:properties], - stream_name: opts[:stream_name], - subscription_id: opts[:subscription_id] - } - } - end - - def new!(%Connection{} = conn, :unsubscribe, opts) do - %Request{ - version: conn.version, - command: :unsubscribe, - correlation_id: conn.correlation_sequence, - data: %UnsubscribeRequestData{ - subscription_id: opts[:subscription_id] - } - } - end - - def new!(%Connection{} = conn, :credit, opts) do - %Request{ - version: conn.version, - command: :credit, - correlation_id: conn.correlation_sequence, - data: %CreditRequestData{ - credit: opts[:credit], - subscription_id: opts[:subscription_id] - } - } - end -end diff --git a/lib/message/response.ex b/lib/message/response.ex deleted file mode 100644 index 74b8a8a..0000000 --- a/lib/message/response.ex +++ /dev/null @@ -1,53 +0,0 @@ -defmodule RabbitMQStream.Message.Response do - @moduledoc false - require Logger - - alias __MODULE__ - - alias RabbitMQStream.{Connection, Message} - - alias Message.Data.{ - TuneData, - CloseData, - HeartbeatData - } - - defstruct([ - :version, - :command, - :correlation_id, - :data, - :code - ]) - - def new!(%Connection{} = conn, :tune, correlation_id: correlation_id) do - %Response{ - version: conn.version, - command: :tune, - correlation_id: correlation_id, - data: %TuneData{ - frame_max: conn.options[:frame_max], - heartbeat: conn.options[:heartbeat] - } - } - end - - def new!(%Connection{} = conn, :heartbeat, correlation_id: correlation_id) do - %Response{ - version: conn.version, - command: :heartbeat, - correlation_id: correlation_id, - data: %HeartbeatData{} - } - end - - def new!(%Connection{} = conn, :close, correlation_id: correlation_id, code: code) do - %Response{ - version: conn.version, - correlation_id: correlation_id, - command: :close, - data: %CloseData{}, - code: code - } - end -end diff --git a/lib/message/osiris.ex b/lib/osiris/chunk.ex similarity index 97% rename from lib/message/osiris.ex rename to lib/osiris/chunk.ex index 76757f8..483b8d9 100644 --- a/lib/message/osiris.ex +++ b/lib/osiris/chunk.ex @@ -210,4 +210,11 @@ defmodule RabbitMQStream.OsirisChunk do data_entries: data_entries } end + + def decode_messages!(chunk, consumer_module) do + %{ + chunk + | data_entries: Enum.map(chunk.data_entries, &consumer_module.decode!/1) + } + end end diff --git a/lib/producer/lifecycle.ex b/lib/producer/lifecycle.ex new file mode 100644 index 0000000..f8696d6 --- /dev/null +++ b/lib/producer/lifecycle.ex @@ -0,0 +1,53 @@ +defmodule RabbitMQStream.Producer.LifeCycle do + @moduledoc false + use GenServer + + # Callbacks + @impl GenServer + def init(opts \\ []) do + reference_name = Keyword.get(opts, :reference_name, Atom.to_string(opts[:producer_module])) + connection = Keyword.get(opts, :connection) || raise(":connection is required") + stream_name = Keyword.get(opts, :stream_name) || raise(":stream_name is required") + + state = %RabbitMQStream.Producer{ + id: nil, + sequence: nil, + stream_name: stream_name, + connection: connection, + reference_name: reference_name, + producer_module: opts[:producer_module] + } + + {:ok, state, {:continue, opts}} + end + + @impl GenServer + def handle_continue(opts, state) do + state = apply(state.producer_module, :before_start, [opts, state]) + + with {:ok, id} <- + RabbitMQStream.Connection.declare_producer(state.connection, state.stream_name, state.reference_name), + {:ok, sequence} <- + RabbitMQStream.Connection.query_producer_sequence(state.connection, state.stream_name, state.reference_name) do + {:noreply, %{state | id: id, sequence: sequence + 1}} + else + err -> + {:stop, err, state} + end + end + + @impl GenServer + def handle_cast({:publish, {message, filter_value}}, %RabbitMQStream.Producer{} = state) when is_binary(message) do + :ok = RabbitMQStream.Connection.publish(state.connection, state.id, state.sequence, message, filter_value) + + {:noreply, %{state | sequence: state.sequence + 1}} + end + + @impl GenServer + def terminate(_reason, %{id: nil}), do: :ok + + def terminate(_reason, state) do + RabbitMQStream.Connection.delete_producer(state.connection, state.id) + :ok + end +end diff --git a/lib/producer/producer.ex b/lib/producer/producer.ex new file mode 100644 index 0000000..c386ce3 --- /dev/null +++ b/lib/producer/producer.ex @@ -0,0 +1,209 @@ +defmodule RabbitMQStream.Producer do + @moduledoc """ + `RabbitMQStream.Producer` allows you to define modules or processes that publish messages to a single stream. + + # Defining a producer Module + + A standalone producer module can be defined with: + + defmodule MyApp.MyProducer do + use RabbitMQStream.Producer, + stream_name: "my-stream", + connection: MyApp.MyConnection + end + + After adding it to your supervision tree, you can publish messages with: + + MyApp.MyProducer.publish("Hello, world!") + + You can add the producer to your supervision tree as follows this: + + def start(_, _) do + children = [ + # ... + MyApp.MyProducer + ] + + opts = # ... + Supervisor.start_link(children, opts) + end + + The standalone producer starts its own `RabbitMQStream.Connection`, declaring itself and fetching its most recent `publishing_id`, and declaring the stream, if it does not exist. + + # Configuration + The RabbitMQStream.Producer accepts the following options: + + * `:stream_name` - The name of the stream to publish to. Required. + * `:reference_name` - The string which is used by the server to prevent [Duplicate Message](https://blog.rabbitmq.com/posts/2021/07/rabbitmq-streams-message-deduplication/). Defaults to `__MODULE__.Producer`. + * `:connection` - The identifier for a `RabbitMQStream.Connection`. + * `:serializer` - The module to use to decode the message. Defaults to `nil`, which means no encoding is done. + + You can also declare the configuration in your `config.exs`: + + config :rabbitmq_stream, MyApp.MyProducer, + stream_name: "my-stream", + connection: MyApp.MyConnection + + + # Setting up + + You can optionally define a `before_start/2` callback to perform setup logic, such as creating the stream, if it doesn't yet exists. + + defmodule MyApp.MyProducer do + use RabbitMQStream.Producer, + stream_name: "my-stream", + connection: MyApp.MyConnection + + @impl true + def before_start(_opts, state) do + # Create the stream + RabbitMQStream.Connection.create_stream(state.connection, state.stream_name) + + state + end + end + + # Configuration + + You can configure each Producer with: + + config :rabbitmq_stream, MyApp.MyProducer, + stream_name: "my-stream", + connection: MyApp.MyConnection + + And also you can override the defaults of all producers with: + + config :rabbitmq_stream, :defaults, + producer: [ + connection: MyApp.MyConnection, + # ... + ] + serializer: Jason + + Globally configuring all producers ignores the following options: + + * `:stream_name` + * `:reference_name` + + """ + + defmacro __using__(opts) do + defaults = Application.get_env(:rabbitmq_stream, :defaults, []) + + serializer = Keyword.get(opts, :serializer, Keyword.get(defaults, :serializer)) + + quote location: :keep do + @opts unquote(opts) + require Logger + + @behaviour RabbitMQStream.Producer + + def start_link(opts \\ []) do + unless !Keyword.has_key?(opts, :serializer) do + Logger.warning("You can only pass `:serializer` option to compile-time options.") + end + + opts = + Application.get_env(:rabbitmq_stream, __MODULE__, []) + |> Keyword.merge(@opts) + |> Keyword.merge(opts) + |> Keyword.put_new(:producer_module, __MODULE__) + |> Keyword.put(:name, __MODULE__) + + RabbitMQStream.Producer.start_link(opts) + end + + def child_spec(opts) do + %{id: __MODULE__, start: {__MODULE__, :start_link, [opts]}} + end + + def publish(message) do + value = filter_value(message) + + message = encode!(message) + + GenServer.cast(__MODULE__, {:publish, {message, value}}) + end + + def before_start(_opts, state), do: state + def filter_value(_), do: nil + + unquote( + # We need this piece of logic so we can garantee that the 'encode!/1' call is executed + # by the caller process, not the Producer process itself. + if serializer != nil do + quote do + def encode!(message), do: unquote(serializer).encode!(message) + end + else + quote do + def encode!(message), do: message + end + end + ) + + defoverridable RabbitMQStream.Producer + end + end + + def start_link(opts \\ []) do + opts = + Application.get_env(:rabbitmq_stream, :defaults, []) + |> Keyword.get(:producer, []) + |> Keyword.drop([:stream_name, :offset_reference, :private]) + |> Keyword.merge(opts) + + GenServer.start_link(RabbitMQStream.Producer.LifeCycle, opts, name: opts[:name]) + end + + def child_spec(opts) do + %{id: __MODULE__, start: {__MODULE__, :start_link, [opts]}} + end + + defstruct [ + :publishing_id, + :reference_name, + :connection, + :stream_name, + :sequence, + :producer_module, + :id + ] + + @type t :: %__MODULE__{ + publishing_id: String.t(), + reference_name: String.t(), + connection: GenServer.server(), + stream_name: String.t(), + sequence: non_neg_integer() | nil, + id: String.t() | nil, + producer_module: module() + } + + @type options :: [option()] + + @type option :: + {:stream_name, String.t()} + | {:reference_name, String.t()} + | {:connection, GenServer.server()} + @optional_callbacks [before_start: 2, filter_value: 1] + + @doc """ + Optional callback that is called after the process has started, but before the + producer has declared itself and fetched its most recent `publishing_id`. + + This is usefull for setup logic, such as creating the Stream, if it doesn't yet exists. + """ + @callback before_start(options(), t()) :: t() + + @doc """ + Callback used to fetch the filter value for a message + + Example: + @impl true + def filter_value(message) do + message["key"] + end + """ + @callback filter_value(term()) :: String.t() +end diff --git a/lib/publisher.ex b/lib/publisher.ex deleted file mode 100644 index c27374d..0000000 --- a/lib/publisher.ex +++ /dev/null @@ -1,211 +0,0 @@ -defmodule RabbitMQStream.Publisher do - @moduledoc """ - `RabbitMQStream.Publisher` allows you to define modules or processes that publish messages to a single stream. - - ## Defining a publisher Module - - A standalone publisher module can be defined with: - - defmodule MyApp.MyPublisher do - use RabbitMQStream.Publisher, - stream_name: "my-stream", - connection: MyApp.MyConnection - end - - After adding it to your supervision tree, you can publish messages with: - - MyApp.MyPublisher.publish("Hello, world!") - - You can add the publisher to your supervision tree as follows this: - - def start(_, _) do - children = [ - # ... - MyApp.MyPublisher - ] - - opts = # ... - Supervisor.start_link(children, opts) - end - - The standalone publisher starts its own `RabbitMQStream.Connection`, declaring itself and fetching its most recent `publishing_id`, and declaring the stream, if it does not exist. - - ## Configuration - The RabbitMQStream.Publisher accepts the following options: - - * `stream_name` - The name of the stream to publish to. Required. - * `reference_name` - The string which is used by the server to prevent [Duplicate Message](https://blog.rabbitmq.com/posts/2021/07/rabbitmq-streams-message-deduplication/). Defaults to `__MODULE__.Publisher`. - * `connection` - The identifier for a `RabbitMQStream.Connection`. - - You can also declare the configuration in your `config.exs`: - - config :rabbitmq_stream, MyApp.MyPublisher, - stream_name: "my-stream", - connection: MyApp.MyConnection - - - ## Setting up - - You can optionally define a `before_start/2` callback to perform setup logic, such as creating the stream, if it doesn't yet exists. - - defmodule MyApp.MyPublisher do - use RabbitMQStream.Publisher, - stream_name: "my-stream", - connection: MyApp.MyConnection - - def before_start(_opts, state) do - # Create the stream - state.connection.create_stream(state.stream_name) - - state - end - end - - ### Configuration - - You can configure each Publisher with: - - config :rabbitmq_stream, MyApp.MyPublisher, - stream_name: "my-stream", - connection: MyApp.MyConnection - - And also you can override the defaults of all publishers with: - - config :rabbitmq_stream, :defaults, - publishers: [ - connection: MyApp.MyConnection, - # ... - ] - - - Globally configuring all publishers ignores the following options: - - * `:stream_name` - * `:reference_name` - - """ - - defmacro __using__(opts) do - quote bind_quoted: [opts: opts] do - use GenServer - @opts opts - - def start_link(opts \\ []) do - opts = - Application.get_env(:rabbitmq_stream, :defaults, []) - |> Keyword.get(:publishers, []) - |> Keyword.drop([:stream_name, :reference_name]) - |> Keyword.merge(Application.get_env(:rabbitmq_stream, __MODULE__, [])) - |> Keyword.merge(@opts) - |> Keyword.merge(opts) - - # opts = Keyword.merge(@opts, opts) - GenServer.start_link(__MODULE__, opts, name: __MODULE__) - end - - def publish(message, sequence \\ nil) when is_binary(message) do - GenServer.cast(__MODULE__, {:publish, message, sequence}) - end - - def stop() do - GenServer.stop(__MODULE__) - end - - ## Callbacks - @impl true - def init(opts \\ []) do - reference_name = Keyword.get(opts, :reference_name, Atom.to_string(__MODULE__)) - connection = Keyword.get(opts, :connection) || raise(":connection is required") - stream_name = Keyword.get(opts, :stream_name) || raise(":stream_name is required") - - state = %RabbitMQStream.Publisher{ - id: nil, - sequence: nil, - stream_name: stream_name, - connection: connection, - reference_name: reference_name - } - - {:ok, state, {:continue, opts}} - end - - @impl true - def handle_continue(opts, state) do - state = - if function_exported?(__MODULE__, :before_start, 2) do - apply(__MODULE__, :before_start, [opts, state]) - else - state - end - - with {:ok, id} <- state.connection.declare_publisher(state.stream_name, state.reference_name), - {:ok, sequence} <- state.connection.query_publisher_sequence(state.stream_name, state.reference_name) do - {:noreply, %{state | id: id, sequence: sequence + 1}} - else - err -> - {:stop, err, state} - end - end - - @impl true - def handle_call({:get_state}, _from, state) do - {:reply, state, state} - end - - @impl true - def handle_cast({:publish, message, nil}, %RabbitMQStream.Publisher{} = publisher) do - publisher.connection.publish(publisher.id, publisher.sequence, message) - - {:noreply, %{publisher | sequence: publisher.sequence + 1}} - end - - @impl true - def terminate(_reason, %{id: nil}), do: :ok - - def terminate(_reason, state) do - state.connection.delete_publisher(state.id) - :ok - end - - if Mix.env() == :test do - def get_state() do - GenServer.call(__MODULE__, {:get_state}) - end - end - end - end - - defstruct [ - :publishing_id, - :reference_name, - :connection, - :stream_name, - :sequence, - :id - ] - - @type t :: %__MODULE__{ - publishing_id: String.t(), - reference_name: String.t(), - connection: RabbitMQStream.Connection.t(), - stream_name: String.t(), - sequence: non_neg_integer() | nil, - id: String.t() | nil - } - - @type options :: [option()] - - @type option :: - {:stream_name, String.t()} - | {:reference_name, String.t()} - | {:connection, RabbitMQStream.Connection.t()} - @optional_callbacks [before_start: 2] - - @doc """ - Optional callback that is called after the process has started, but before the - publisher has declared itself and fetched its most recent `publishing_id`. - - This is usefull for setup logic, such as creating the Stream, if it doesn't yet exists. - """ - @callback before_start(options(), t()) :: t() -end diff --git a/lib/subscriber/subscriber.ex b/lib/subscriber/subscriber.ex deleted file mode 100644 index bc4adb7..0000000 --- a/lib/subscriber/subscriber.ex +++ /dev/null @@ -1,304 +0,0 @@ -defmodule RabbitMQStream.Subscriber do - @moduledoc """ - Used to declare a Persistent Subscriber module. It is able to process - chunks by implementing the `handle_chunk/1` or `handle_chunk/2` callbacks. - - ## Usage - - defmodule MyApp.MySubscriber do - use RabbitMQStream.Subscriber, - connection: MyApp.MyConnection, - stream_name: "my_stream", - initial_offset: :first - - @impl true - def handle_chunk(%RabbitMQStream.OsirisChunk{} = _chunk, _subscriber) do - :ok - end - end - - - ## Parameters - - * `:connection` - The connection module to use. This is required. - * `:stream_name` - The name of the stream to subscribe to. This is required. - * `:initial_offset` - The initial offset to subscribe from. This is required. - * `:initial_credit` - The initial credit to request from the server. Defaults to `50_000`. - * `:offset_tracking` - Offset tracking strategies to use. Defaults to `[count: [store_after: 50]]`. - * `:flow_control` - Flow control strategy to use. Defaults to `[count: [credit_after: {:count, 1}]]`. - * `:private` - Private data that can hold any value, and is passed to the `handle_chunk/2` callback. - - - ## Offset Tracking - - The subscriber is able to track its progress in the stream by storing its - latests offset in the stream. Check [Offset Tracking with RabbitMQ Streams(https://blog.rabbitmq.com/posts/2021/09/rabbitmq-streams-offset-tracking/) for more information on - how offset tracking works. - - The subscriber can be configured to use different offset tracking strategies, - which decide when to store the offset in the stream. You can implement your - own strategy by implementing the `RabbitMQStream.Subscriber.OffsetTracking.Strategy` - behaviour, and passing it to the `:offset_tracking` option. It defaults to - `RabbitMQStream.Subscriber.OffsetTracking.CountStrategy`, which stores the - offset after, by default, every 50_000 messages. - - ## Flow Control - - The RabbitMQ Streams server requires that the subscriber declares how many - messages it is able to process at a time. This is done by informing an amount - of 'credits' to the server. After every chunk is sent, one credit is consumed, - and the server will send messages only if there are credits available. - - We can configure the subscriber to automatically request more credits based on - a strategy. By default it uses the `RabbitMQStream.Subscriber.FlowControl.MessageCount`, - which requests 1 additional credit for every 1 processed chunk. Please check - the RabbitMQStream.Subscriber.FlowControl.MessageCount module for more information. - - You can also call `RabbitMQStream.Subscriber.credit/2` to manually add more - credits to the subscription, or implement your own strategy by implementing - the `RabbitMQStream.Subscriber.FlowControl.Strategy` behaviour, and passing - it to the `:flow_control` option. - - You can find more information on the [RabbitMQ Streams documentation](https://www.rabbitmq.com/stream.html#flow-control). - - If you want an external process to be fully in control of the flow control - of a subscriber, you can set the `:flow_control` option to `false`. Then - you can call `RabbitMQStream.Subscriber.credit/2` to manually add more - credits to the subscription. - - - ## Configuration - - You can configure each subscriber with: - - config :rabbitmq_stream, MyApp.MySubscriber, - connection: MyApp.MyConnection, - stream_name: "my_stream", - initial_offset: :first, - initial_credit: 50_000, - offset_tracking: [count: [store_after: 50]], - flow_control: [count: [credit_after: {:count, 1}]] - - These options are overriden by the options passed to the `use` macro, which - are overriden by the options passed to `start_link/1`. - - And also you can override the defaults of all subscribers with: - - config :rabbitmq_stream, :defaults, - subscribers: [ - connection: MyApp.MyConnection, - initial_credit: 50_000, - # ... - ] - Globally configuring all subscribers ignores the following options: - - * `:stream_name` - * `:offset_reference` - * `:private` - - """ - - defmacro __using__(opts) do - quote bind_quoted: [opts: opts], location: :keep do - use GenServer - @behaviour RabbitMQStream.Subscriber - alias RabbitMQStream.Subscriber.{FlowControl, OffsetTracking} - - @opts opts - - def credit(amount) do - GenServer.cast(__MODULE__, {:credit, amount}) - end - - def get_credits() do - GenServer.call(__MODULE__, :get_credits) - end - - def start_link(opts \\ []) do - opts = - Application.get_env(:rabbitmq_stream, :defaults, []) - |> Keyword.get(:subscribers, []) - |> Keyword.drop([:stream_name, :offset_reference, :private]) - |> Keyword.merge(Application.get_env(:rabbitmq_stream, __MODULE__, [])) - |> Keyword.merge(@opts) - |> Keyword.merge(opts) - - # opts = Keyword.merge(@opts, opts) - GenServer.start_link(__MODULE__, opts, name: __MODULE__) - end - - @impl true - def init(opts \\ []) do - connection = Keyword.get(opts, :connection) || raise(":connection is required") - stream_name = Keyword.get(opts, :stream_name) || raise(":stream_name is required") - initial_credit = Keyword.get(opts, :initial_credit, 50_000) - - offset_tracking = Keyword.get(opts, :offset_tracking, count: [store_after: 50]) - flow_control = Keyword.get(opts, :flow_control, count: [credit_after: {:count, 1}]) - - offset_reference = Keyword.get(opts, :offset_reference, Atom.to_string(__MODULE__)) - - state = %RabbitMQStream.Subscriber{ - stream_name: stream_name, - connection: connection, - offset_reference: offset_reference, - private: opts[:private], - offset_tracking: OffsetTracking.Strategy.init(offset_tracking, opts), - flow_control: FlowControl.Strategy.init(flow_control, opts), - credits: initial_credit, - initial_credit: initial_credit - } - - {:ok, state, {:continue, opts}} - end - - @impl true - def handle_continue(opts, state) do - initial_offset = Keyword.get(opts, :initial_offset) || raise(":initial_offset is required") - - last_offset = - case state.connection.query_offset(state.stream_name, state.offset_reference) do - {:ok, offset} -> - {:offset, offset} - - _ -> - initial_offset - end - - case state.connection.subscribe(state.stream_name, self(), last_offset, state.initial_credit) do - {:ok, id} -> - last_offset = - case last_offset do - {:offset, offset} -> - offset - - _ -> - nil - end - - {:noreply, %{state | id: id, last_offset: last_offset}} - - err -> - {:stop, err, state} - end - end - - @impl true - def terminate(_reason, state) do - state.connection.unsubscribe(state.id) - :ok - end - - @impl true - def handle_info({:chunk, %RabbitMQStream.OsirisChunk{} = chunk}, state) do - cond do - function_exported?(__MODULE__, :handle_chunk, 1) -> - apply(__MODULE__, :handle_chunk, [chunk]) - - function_exported?(__MODULE__, :handle_chunk, 2) -> - apply(__MODULE__, :handle_chunk, [chunk, state]) - - true -> - raise "handle_chunk/1 or handle_chunk/2 must be implemented" - end - - offset_tracking = - for {strategy, track_state} <- state.offset_tracking do - if function_exported?(strategy, :after_chunk, 3) do - {strategy, strategy.after_chunk(track_state, chunk, state)} - else - {strategy, track_state} - end - end - - credit = state.credits - chunk.num_entries - - state = - %{state | offset_tracking: offset_tracking, last_offset: chunk.chunk_id, credits: credit} - - state = state |> OffsetTracking.Strategy.run() |> FlowControl.Strategy.run() - - {:noreply, state} - end - - def handle_info(:run_offset_tracking, state) do - {:noreply, OffsetTracking.Strategy.run(state)} - end - - def handle_info(:run_flow_control, state) do - {:noreply, FlowControl.Strategy.run(state)} - end - - @impl true - def handle_cast({:credit, amount}, state) do - state.connection.credit(state.id, amount) - {:noreply, %{state | credits: state.credits + amount}} - end - - @impl true - def handle_call(:get_credits, _from, state) do - {:reply, state.credits, state} - end - end - end - - @optional_callbacks handle_chunk: 1, handle_chunk: 2 - - @doc """ - The callback that is invoked when a chunk is received. - - Each chunk contains a list of potentially many data entries, along with - metadata about the chunk itself. The callback is invoked once for each - chunk received. - - Optionally if you implement `handle_chunk/2`, it also passes the current - state of the subscriber. It can be used to access the `private` field - passed to `start_link/1`, and other fields. - - The return value is ignored. - """ - @callback handle_chunk(chunk :: RabbitMQStream.OsirisChunk.t()) :: term() - @callback handle_chunk(chunk :: RabbitMQStream.OsirisChunk.t(), state :: t()) :: term() - - defstruct [ - :offset_reference, - :connection, - :stream_name, - :offset_tracking, - :flow_control, - :id, - :last_offset, - # We could have delegated the tracking of the credit to the strategy, - # by adding declaring a callback similar to `after_chunk/3`. But it seems - # reasonable to have a `credit` function to manually add more credits, - # which would them possibly cause the strategy to not work as expected. - :credits, - :initial_credit, - :private - ] - - @type t :: %__MODULE__{ - offset_reference: String.t(), - connection: RabbitMQStream.Connection.t(), - stream_name: String.t(), - id: non_neg_integer() | nil, - offset_tracking: [{RabbitMQStream.Subscriber.OffsetTracking.Strategy.t(), term()}], - flow_control: {RabbitMQStream.Subscriber.FlowControl.Strategy.t(), term()}, - last_offset: non_neg_integer() | nil, - private: any(), - credits: non_neg_integer(), - initial_credit: non_neg_integer() - } - - @type subscriber_option :: - {:offset_reference, String.t()} - | {:connection, RabbitMQStream.Connection.t()} - | {:stream_name, String.t()} - | {:initial_offset, RabbitMQStream.Connection.offset()} - | {:initial_credit, non_neg_integer()} - | {:offset_tracking, [{RabbitMQStream.Subscriber.OffsetTracking.Strategy.t(), term()}]} - | {:flow_control, {RabbitMQStream.Subscriber.FlowControl.Strategy.t(), term()}} - | {:private, any()} - - @type opts :: [subscriber_option()] -end diff --git a/lib/super_consumer/manager.ex b/lib/super_consumer/manager.ex new file mode 100644 index 0000000..35cb3ec --- /dev/null +++ b/lib/super_consumer/manager.ex @@ -0,0 +1,41 @@ +defmodule RabbitMQStream.SuperConsumer.Manager do + alias RabbitMQStream.SuperConsumer + + use GenServer + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts) + end + + @impl true + def init(opts \\ []) do + state = struct(SuperConsumer, opts) + + {:ok, state, {:continue, :start}} + end + + @impl true + def handle_continue(:start, %SuperConsumer{} = state) do + for partition <- 0..(state.partitions - 1) do + {:ok, _pid} = + DynamicSupervisor.start_child( + state.dynamic_supervisor, + { + RabbitMQStream.Consumer, + Keyword.merge(state.consumer_opts, + name: {:via, Registry, {state.registry, partition}}, + stream_name: "#{state.super_stream}-#{partition}", + consumer_module: state.consumer_module, + properties: [ + single_active_consumer: true, + # It might not be necessary to set the super_stream as of 3.11 + super_stream: state.super_stream + ] + ) + } + ) + end + + {:noreply, state, :hibernate} + end +end diff --git a/lib/super_consumer/super_consumer.ex b/lib/super_consumer/super_consumer.ex new file mode 100644 index 0000000..8ef7153 --- /dev/null +++ b/lib/super_consumer/super_consumer.ex @@ -0,0 +1,97 @@ +defmodule RabbitMQStream.SuperConsumer do + @moduledoc """ + A Superconsumer spawns a Consumer process for each partition of the stream. + + It accepts the same options as a Consumer, plus the following: + + * `:super_stream` - the name of the super stream + * `:partitions` - the number of partitions + + + All the consumers use the same provided connection, and are supervised by a + DynamicSupervisor. + + """ + + defmacro __using__(opts) do + defaults = Application.get_env(:rabbitmq_stream, :defaults, []) + + serializer = Keyword.get(opts, :serializer, Keyword.get(defaults, :serializer)) + opts = Keyword.put_new(opts, :partitions, Keyword.get(defaults, :partitions, 1)) + + quote do + @opts unquote(opts) + @behaviour RabbitMQStream.Consumer + + use Supervisor + + def start_link(opts) do + opts = Keyword.merge(opts, @opts) + Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + def child_spec(opts) do + %{id: __MODULE__, start: {__MODULE__, :start_link, [opts]}} + end + + @impl true + def init(opts) do + {opts, consumer_opts} = Keyword.split(opts, [:super_stream]) + + children = [ + {Registry, keys: :unique, name: __MODULE__.Registry}, + {DynamicSupervisor, strategy: :one_for_one, name: __MODULE__.DynamicSupervisor}, + {RabbitMQStream.SuperConsumer.Manager, + opts ++ + [ + name: __MODULE__.Manager, + dynamic_supervisor: __MODULE__.DynamicSupervisor, + registry: __MODULE__.Registry, + consumer_module: __MODULE__, + partitions: @opts[:partitions], + consumer_opts: consumer_opts + ]} + ] + + Supervisor.init(children, strategy: :one_for_all) + end + + def before_start(_opts, state), do: state + + unquote( + if serializer != nil do + quote do + def decode!(message), do: unquote(serializer).decode!(message) + end + else + quote do + def decode!(message), do: message + end + end + ) + end + end + + defstruct [ + :super_stream, + :partitions, + :registry, + :dynamic_supervisor, + :consumer_module, + :consumer_opts + ] + + @type t :: %__MODULE__{ + super_stream: String.t(), + partitions: non_neg_integer(), + dynamic_supervisor: module(), + consumer_module: module(), + registry: module(), + consumer_opts: [RabbitMQStream.Consumer.consumer_option()] | nil + } + + @type super_consumer_option :: + {:super_stream, String.t()} + | {:partitions, non_neg_integer()} + | RabbitMQStream.Consumer.consumer_option() +end diff --git a/lib/super_producer/manager.ex b/lib/super_producer/manager.ex new file mode 100644 index 0000000..240c99b --- /dev/null +++ b/lib/super_producer/manager.ex @@ -0,0 +1,36 @@ +defmodule RabbitMQStream.SuperProducer.Manager do + alias RabbitMQStream.SuperProducer + + use GenServer + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts) + end + + @impl true + def init(opts \\ []) do + state = struct(SuperProducer, opts) + + {:ok, state, {:continue, :start}} + end + + @impl true + def handle_continue(:start, %SuperProducer{} = state) do + for partition <- 0..(state.partitions - 1) do + {:ok, _pid} = + DynamicSupervisor.start_child( + state.dynamic_supervisor, + { + RabbitMQStream.Producer, + Keyword.merge(state.producer_opts, + name: {:via, Registry, {state.registry, partition}}, + stream_name: "#{state.super_stream}-#{partition}", + producer_module: state.producer_module + ) + } + ) + end + + {:noreply, state, :hibernate} + end +end diff --git a/lib/super_producer/super_producer.ex b/lib/super_producer/super_producer.ex new file mode 100644 index 0000000..fb04f72 --- /dev/null +++ b/lib/super_producer/super_producer.ex @@ -0,0 +1,137 @@ +defmodule RabbitMQStream.SuperProducer do + @moduledoc """ + A Superproducer spawns a Producer process for each partition of the stream, + and uses the `partition/2` callback to forward a publish command to the + producer of the partition. + + It accepts the same options as a Producer, plus the following: + + * `:super_stream` - the name of the super stream + * `:partitions` - the number of partitions + + All the producers use the same provided connection, and are supervised by a + DynamicSupervisor. + + You can optionally implement a `partition/2` callback to compute the target + partition for a given message. By default, the partition is computed using + `:erlang.phash2/2`. + + + ## Setup + + To start a Superproducer, you need to make sure that each stream/partition + is created beforehand. As of RabbitMQ 3.11.x and 3.12.x, this can only be done + using an [AMQP Client, RabbitMQ Management or with the RabbitMQ CLI.](https://www.rabbitmq.com/streams.html#super-streams). + + The easiest way to do this is to use the RabbitMQ CLI: + + `$ rabbitmq-streams add_super_stream invoices --partitions 3` + + As of RabbitMQ 3.13.x, you can also create a super stream using the + `RabbitMQStream.Connection.create_super_stream/4`. + + """ + defmacro __using__(opts) do + defaults = Application.get_env(:rabbitmq_stream, :defaults, []) + + serializer = Keyword.get(opts, :serializer, Keyword.get(defaults, :serializer)) + opts = Keyword.put_new(opts, :partitions, Keyword.get(defaults, :partitions, 1)) + + quote do + @opts unquote(opts) + @behaviour RabbitMQStream.Producer + + use Supervisor + + def start_link(opts) do + opts = Keyword.merge(opts, @opts) + Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + @impl true + def init(opts) do + {opts, producer_opts} = Keyword.split(opts, [:super_stream]) + + children = [ + {Registry, keys: :unique, name: __MODULE__.Registry}, + {DynamicSupervisor, strategy: :one_for_one, name: __MODULE__.DynamicSupervisor}, + {RabbitMQStream.SuperProducer.Manager, + opts ++ + [ + name: __MODULE__.Manager, + dynamic_supervisor: __MODULE__.DynamicSupervisor, + registry: __MODULE__.Registry, + producer_module: __MODULE__, + partitions: @opts[:partitions], + producer_opts: producer_opts + ]} + ] + + Supervisor.init(children, strategy: :one_for_all) + end + + def publish(message) do + value = filter_value(message) + + message = encode!(message) + + partition = partition(message, @opts[:partitions]) + + GenServer.cast( + {:via, Registry, {__MODULE__.Registry, partition}}, + {:publish, {message, value}} + ) + end + + def stop() do + GenServer.stop(__MODULE__) + end + + def partition(message, partitions) do + :erlang.phash2(message, partitions) + end + + def before_start(_opts, state), do: state + def filter_value(_), do: nil + + unquote( + # We need this piece of logic so we can garantee that the 'encode!/1' call is executed + # by the caller process, not the Producer process itself. + if serializer != nil do + quote do + def encode!(message), do: unquote(serializer).encode!(message) + end + else + quote do + def encode!(message), do: message + end + end + ) + + defoverridable RabbitMQStream.Producer + end + end + + defstruct [ + :super_stream, + :partitions, + :dynamic_supervisor, + :producer_module, + :registry, + :producer_opts + ] + + @type t :: %__MODULE__{ + super_stream: String.t(), + partitions: non_neg_integer(), + dynamic_supervisor: module(), + producer_module: module(), + registry: module(), + producer_opts: [RabbitMQStream.Producer.producer_option()] | nil + } + + @type super_producer_option :: + {:super_stream, String.t()} + | {:partitions, non_neg_integer()} + | RabbitMQStream.Producer.producer_option() +end diff --git a/mix.exs b/mix.exs index bf4109e..315fb4a 100644 --- a/mix.exs +++ b/mix.exs @@ -33,14 +33,15 @@ defmodule RabbitMQStream.MixProject do # Run "mix help compile.app" to learn about applications. def application do [ - extra_applications: [:logger] + extra_applications: [:logger, :crypto, :ssl] ] end # Run "mix help deps" to learn about dependencies. defp deps do [ - {:ex_doc, "~> 0.28.4", only: :dev, runtime: false} + {:ex_doc, "~> 0.28.4", only: :dev, runtime: false}, + {:jason, "~> 1.4.1", only: :test, runtime: false} ] end @@ -68,17 +69,20 @@ defmodule RabbitMQStream.MixProject do defp extras do [ - "guides/introduction/getting-started.md", - "guides/tutorial/subscribing.md", + "guides/concepts/streams.md", + "guides/concepts/super-streams.md", + "guides/concepts/single-active-consumer.md", + "guides/concepts/offset.md", + "guides/setup/getting-started.md", + "guides/setup/configuration.md", "CHANGELOG.md" ] end defp groups_for_extras do [ - Introduction: ~r/guides\/introduction\/.*/, - Tutorial: ~r/guides\/tutorial\/.*/, - Topics: ~r/guides\/[^\/]+\.md/, + Concepts: ~r/guides\/concepts\/.*/, + Setup: ~r/guides\/setup\/.*/, Changelog: "CHANGELOG.md" ] end @@ -87,17 +91,17 @@ defmodule RabbitMQStream.MixProject do [ Client: [ RabbitMQStream.Connection, - RabbitMQStream.Publisher, - RabbitMQStream.Subscriber + RabbitMQStream.Producer, + RabbitMQStream.Consumer ], "Offset Tracking": [ - RabbitMQStream.Subscriber.OffsetTracking.Strategy, - RabbitMQStream.Subscriber.OffsetTracking.CountStrategy, - RabbitMQStream.Subscriber.OffsetTracking.IntervalStrategy + RabbitMQStream.Consumer.OffsetTracking, + RabbitMQStream.Consumer.OffsetTracking.CountStrategy, + RabbitMQStream.Consumer.OffsetTracking.IntervalStrategy ], "Flow Control": [ - RabbitMQStream.Subscriber.FlowControl.Strategy, - RabbitMQStream.Subscriber.FlowControl.MessageCount + RabbitMQStream.Consumer.FlowControl, + RabbitMQStream.Consumer.FlowControl.MessageCount ] ] end diff --git a/mix.lock b/mix.lock index f8baec8..d290b1a 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,7 @@ %{ "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "ex_doc": {:hex, :ex_doc, "0.28.6", "2bbd7a143d3014fc26de9056793e97600ae8978af2ced82c2575f130b7c0d7d7", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bca1441614654710ba37a0e173079273d619f9160cbcc8cd04e6bd59f1ad0e29"}, + "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"}, diff --git a/services/cert/ca_certificate.pem b/services/cert/ca_certificate.pem new file mode 100755 index 0000000..ac4692d --- /dev/null +++ b/services/cert/ca_certificate.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDhjCCAm6gAwIBAgIUGoiiuRDW3uugpTmO8k1Zp6vGFLEwDQYJKoZIhvcNAQEL +BQAwTDE7MDkGA1UEAwwyVExTR2VuU2VsZlNpZ25lZHRSb290Q0EgMjAyNC0wMS0w +NlQyMDo0NDo0MS45MTQ1OTMxDTALBgNVBAcMBCQkJCQwHhcNMjQwMTA2MjM0NDQx +WhcNMzQwMTAzMjM0NDQxWjBMMTswOQYDVQQDDDJUTFNHZW5TZWxmU2lnbmVkdFJv +b3RDQSAyMDI0LTAxLTA2VDIwOjQ0OjQxLjkxNDU5MzENMAsGA1UEBwwEJCQkJDCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMapfVUuVsyhMMG+AcCe+mDE +zEImAQ15P01HiFYuYBNyxFl+LmRrJRBbkSdUvGJLBXgbje0i1ehcKmNzLg9enk0T +rzkHCtIJ7YJszKMh7tTFlbpYyjEKkefcEp45HGtHeHlMY5mKeih6pbZTjBlutR4g +zf5LvknrRJ9cgCqVhjFnvk6PQqAU+bIEH+ms32mIsMSj6BxHolSbtgKCAxDsouGB +ILYHCKayqPSMo56JbS+nSheSgviCYp0wH09JCZhIcfNFV0wZNzYl21ufJUy9rz+E +h3T0hm9a4/dnjhCmJFf4ZsbnLriKl6aQlFdPwMzWpUWqtZD7WegBhDwNTzvd6dsC +AwEAAaNgMF4wDwYDVR0TAQH/BAUwAwEB/zALBgNVHQ8EBAMCAQYwHQYDVR0OBBYE +FOebi8sSuBv4EBdPRxYIyAJYa2kEMB8GA1UdIwQYMBaAFOebi8sSuBv4EBdPRxYI +yAJYa2kEMA0GCSqGSIb3DQEBCwUAA4IBAQAAkMp7oBWY7VGSKR3P2OUvfwsxQhKS +5+ugB8L0GyBzlFgciZo4am5cooX6BJhBPPX0pS0+CCESlmKodtbL90S9Uc0gQoRs +e1NXSCC9hA/XTjMExTUGsFdNMMjT9cSPnNdv1rv7LUQ/ZPmD05t1j4REoq3r4Jk2 +TD4Uk08SduK2iPWtpRqj90KBYYRYTjGajfDzcH0SlEiVfc3Tu/uqzbAoY2i5g5J4 +tH/jBjYxCEnpE5sGgH+EYbtD4tVolK0hrCyS8X7F0l/xtJcVG6pLN0hYXxlBXtPJ +fq53xNIbCRLl8dD3589fQqC+qM6M3Ty9o1GVosfvg0EnbN8RMMZF1Pew +-----END CERTIFICATE----- diff --git a/services/cert/ca_key.pem b/services/cert/ca_key.pem new file mode 100755 index 0000000..6d65695 --- /dev/null +++ b/services/cert/ca_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDGqX1VLlbMoTDB +vgHAnvpgxMxCJgENeT9NR4hWLmATcsRZfi5kayUQW5EnVLxiSwV4G43tItXoXCpj +cy4PXp5NE685BwrSCe2CbMyjIe7UxZW6WMoxCpHn3BKeORxrR3h5TGOZinooeqW2 +U4wZbrUeIM3+S75J60SfXIAqlYYxZ75Oj0KgFPmyBB/prN9piLDEo+gcR6JUm7YC +ggMQ7KLhgSC2Bwimsqj0jKOeiW0vp0oXkoL4gmKdMB9PSQmYSHHzRVdMGTc2Jdtb +nyVMva8/hId09IZvWuP3Z44QpiRX+GbG5y64ipemkJRXT8DM1qVFqrWQ+1noAYQ8 +DU873enbAgMBAAECggEAAPz62J+gYugyW+RcEd84k56m/XU+8Bg4TmQas4MLi004 +2NEd3fYCezBZan6raTnCTzSj4hgYRE2hEFmUUxM/ala8umXkkeLmwY1U/0K982UA +N6sVZmUNEMAxf4P9NjNm0AOnQy5DKUxi4qNLwRyh3gJ/w9IQokF/V+OX555SypAd +ShTVGa4E0buV+Yl4oVNBQvqeZL7G59IiRTwRuvMSwPW694M1cCuJ2p8qzd2NNdoY +zoMQfzC7Gd1S8wiOktXoDy/Vzol9GdoI0GTFSovSkEs6A+aEYXvI9v6hlMl1d0zQ +Y29U6oHeHHebDV1lwRGqcvFDg1eoh9wICMGP8U2hQQKBgQD88TYFY7NzLxRG+Ifa +DAbtMP7yAdTuQn7ZdsnugLdbhPJ1ni43QOZvKhxdfgnmGeT4BAoLNs8U5gbt1EPi +fZjnxzIqPf5xGXV5z16qGfP9m0ZHRjTmaldG9sOC7UYzjRJbdF0/hnrtvOaLWEqd +bWPA91tSJFVBprrjB2aBWUYoSwKBgQDJEEu/MIIdesQaBf8ySS4QWaC+EBsG1y+i +7mWnZSwWpf48LbJgzkH/jfSmsLWm9uRbBX/Spgl0gv+4RHBCR9rP/QqK/vPLV3p1 +oP7D+vGoIHtCcEsu8Hd7aJ1PIPzMEuySMcbpCi6WzZ3ZHhP3B/QOO7BnqPsBE0na +spzVDudqsQKBgEcWt9sN6VpfCfDkWrISnUO+eHilwSVxdNtDgn9Ql7fWBpq96TlI +OTtW93/jM38DGhIGeJgsQEkcWSgwdx/Jsta5akTrBX7d6+FfQbjG8Ib/Q+I2Phng +G0VrhwleDFPiux0O+EIpVpVIePcCyn2yR83s9zJ/2aJI7M9vvgRuhcQ/AoGAQdW/ +J6wnfqWbHnZGOF3z4lCmrHUzlErTg0MSL/yVshjKJURFOyNuQtJlgEsuP6xp20/y +qbPKNsdKGjj2lQ1YHXBaimauxy8unuOHZ/58MDPqiDeitozwYo0/rRA9FklAAyKf +YeD+nkxXWidaHDITfLGYsmiHP7PkI+MGLVFC/xECgYEAuhAWoP06PlEQ1IJyGKPy +z2Y1ZFWcSqvtqAncV5PBwALtAmvURYFdkVCXeNxNXQ9wEJaJFm42l+gcdcLgSiTW +GFnI9C3hbt95TKcCWtVxJWZiy20WrdwG1GT0AvjYP7PBXPHQYvekclVn33zXSuWW +zkjRZ75CDy1WHmf8IXCPA9o= +-----END PRIVATE KEY----- diff --git a/services/cert/client_box_certificate.pem b/services/cert/client_box_certificate.pem new file mode 100755 index 0000000..ed1dbd9 --- /dev/null +++ b/services/cert/client_box_certificate.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDqjCCApKgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBMMTswOQYDVQQDDDJUTFNH +ZW5TZWxmU2lnbmVkdFJvb3RDQSAyMDI0LTAxLTA2VDIwOjQ0OjQxLjkxNDU5MzEN +MAsGA1UEBwwEJCQkJDAeFw0yNDAxMDYyMzQ0NDJaFw0zNDAxMDMyMzQ0NDJaMB8x +DDAKBgNVBAMMA2JveDEPMA0GA1UECgwGY2xpZW50MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAt+La1QPCHCsKHyeCwAsqzUj5mtTm1YVsykG476YpbwtF +Grlpb0GKoX8swXJ6ZEh8MS+uD8NWTYkFPK95vbJw+TWZIl9qQNiC3RJ6LEt1xdFT +nycmdmzSqrGPMVO1ypgjUqRklVkF3UfVfsO/rH0zi3tYIm9ZqOUyk03ruyJ+0IeK +nQ9gHWu3T2OrpPdDSoOUaCu39MKW8oB5zxyzYFgT+6IRcRmOTAYzitkJbVMDzf66 +JdunoApr44Y0KBNqnAYNlt36UeR6Q5+4sLV04lzUoGEoDXA4mygEnQsPrvMRGGmk +7zPb4f3VDnBjUHzBnKigxSkCAL1CTTLF+QZzE5hAzwIDAQABo4HDMIHAMAkGA1Ud +EwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMCMB4GA1UdEQQX +MBWCA2JveIIDYm94gglsb2NhbGhvc3QwMQYDVR0fBCowKDAmoCSgIoYgaHR0cDov +L2NybC1zZXJ2ZXI6ODAwMC9iYXNpYy5jcmwwHQYDVR0OBBYEFD0161Lvfskd+d46 +SBRo522SEPUkMB8GA1UdIwQYMBaAFOebi8sSuBv4EBdPRxYIyAJYa2kEMA0GCSqG +SIb3DQEBCwUAA4IBAQCr5Cg+Ps3pNoe/eerj2mp77OOeg/wzTnuiyAGxIS2HM/yK +FaMq/dNxDDOYcdm79BZ57gSprFMLkRSmahF/yGB2HQ9CWrZ0ZRemAFjgIqpBUIoe +ap2vJoectc4qAYdyoPhL6TV0gpnCWKecIUOOEggg0zc4uXNiRKZYtYkp0BVy3Om9 +kayW71qOozPQGuJayNRq0MzhlXh0S0UqKtxJZBAvOjQUcsHXu4Y16nEOXOQPuzBB +FBESowqN1SRaxxEVFxEX5DHttIei+Kwymcd39yX3iQWeJzkujVm3PzlJv2VGZ3kZ +KKX/H8pRlTH3ytiid7RDHM8m7pAdgsI20jSPWqFY +-----END CERTIFICATE----- diff --git a/services/cert/client_box_key.pem b/services/cert/client_box_key.pem new file mode 100755 index 0000000..6a6e15a --- /dev/null +++ b/services/cert/client_box_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC34trVA8IcKwof +J4LACyrNSPma1ObVhWzKQbjvpilvC0UauWlvQYqhfyzBcnpkSHwxL64Pw1ZNiQU8 +r3m9snD5NZkiX2pA2ILdEnosS3XF0VOfJyZ2bNKqsY8xU7XKmCNSpGSVWQXdR9V+ +w7+sfTOLe1gib1mo5TKTTeu7In7Qh4qdD2Ada7dPY6uk90NKg5RoK7f0wpbygHnP +HLNgWBP7ohFxGY5MBjOK2QltUwPN/rol26egCmvjhjQoE2qcBg2W3fpR5HpDn7iw +tXTiXNSgYSgNcDibKASdCw+u8xEYaaTvM9vh/dUOcGNQfMGcqKDFKQIAvUJNMsX5 +BnMTmEDPAgMBAAECggEAIu/XetxFb3uf5y5nEZyti6Y/QjMMDmpUspLNE78ipSXj +gcHTbd9qeueCNdjNtyrsaZ3w+K64wW90jsSaITf7beXIi/bL5bkdF+qQSsUlgamV +NSpHjP19AxBML+eDJRRFtchlEdkBm7qvre98BcYd2nTbWGOGiefN9Z262Wmi5XRf +jMN2VIqaKDGd/bZepBVX2sxvIhQF99FCNk4aObMuw89pjAAStdyOKqkJcETeW0Er +Yl7wRnC5InFNoNd9ACnHEOBU61Yc3ViukyGk98Ejw8IkjH1J6GWMMiReROGJb/pi +L3zGAnudEYb9Q4hk2f2pB0U1ApVQdwuDIBLDq82BfQKBgQD/jM31DWlYVLi2JQOz +AFffDvQj0JWuNY882G3R1EG04Wl9OJwCvSkRwQPiaiBVU4PDhzou5YkNB8P1bNlg +YXk8mnPRenft6zLWvk3i0hVHg/T7ayM3AAidkBpwvZpvXU5/jPJuKYKSAw+h2MEP ++Auue8UoT41CL1L7bKHaKxhEZQKBgQC4Nb78LwEnZmfZHDqm7VAXFjT1bg3Cl7l4 +GjwbAWyEhXeoyECM3W1wwQoFEoI5iCwvjIAsCwcZnBYgoP10G8GOx8YDt6XybO7H +5sim2gKANjy8lJyAs+Cm94vRcVW0qoc8VMeM4kwrLpM5V0wBXavcteuUiItgm0oN +BtQjqitbIwKBgBVlRY0hzVMe7MQbPz8KZVEAoIIrIY0PYOm4OSGbQtERGlLonW9B +RSH+ZgPb3M8oCd0iAkYQ47OaldaDgYOnRY5EZDQCq+3Yhk4iasT1z+BH9QfjxXQC +ZbZDbWNJDJgMWNknBNkiUpNE/FcRncBnhRrbs/sXBf2nlqwEQVzRxNtdAoGANwj3 +smhos9jIlfj1HQIxt3QPBnSG7+hcpSFRa9AVF9K4WVja0Dsng1RpJfLNrVqduOcF +NCNnT6NuiPkQQCw6u+m7o0iu76rT/C1bLV5c+Ok7ZCwSRfF34Op8f2qY7I02MjQy +GklR8GTN34fRUWcm/Z3scEgLPCWpDhNINg+VPHkCgYAV9jMU2L9blN40z/ek7H6s +fyqQq1kqZc3xmd//sbGHBPiiqllO2tBAz0A4M32hnvcIkf+nqalRHSfDxXm6BLNg +Q91K7Sgwo0Zs8i7jmMFA7AogBkdKTLDAuXqhMTFYCis6bKEOr6lya/JP7QFOtRHy +T4UEORbVqoQpYSqLohP4KA== +-----END PRIVATE KEY----- diff --git a/services/cert/server_box_certificate.pem b/services/cert/server_box_certificate.pem new file mode 100755 index 0000000..f75ac46 --- /dev/null +++ b/services/cert/server_box_certificate.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDqjCCApKgAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMTswOQYDVQQDDDJUTFNH +ZW5TZWxmU2lnbmVkdFJvb3RDQSAyMDI0LTAxLTA2VDIwOjQ0OjQxLjkxNDU5MzEN +MAsGA1UEBwwEJCQkJDAeFw0yNDAxMDYyMzQ0NDFaFw0zNDAxMDMyMzQ0NDFaMB8x +DDAKBgNVBAMMA2JveDEPMA0GA1UECgwGc2VydmVyMIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAwE2lePhgR07kXujUSBqvkUvZmUk9nuJiqUIIxqrixoX3 +xTLcaEZIeCluCwOURdnAISJFxQQmVe1U+6NFOOXCsLuAWC/7z11OPX6b4t4opaHW +cUmeEVJjuxxIlzD+14MWZc+UYACQMdDSFVn4bS3r4L4h/CCeZzy3OPfNW/mognGQ +wTQHrlWf6hFBXtB0itZKuvQb3NFvdEBqVUyWNvxc5wR/1D3xJ1WREw+wMzozG3wd +DfebTA97FymalYKmr82kXu9BJ62/MVX/Kx5QnG1u7hSApF9SF0D2q6JWMain07m4 +k1zz9J++C3eCTeYdz4+8QWiwlu7TjwUwyQ6BBchUnwIDAQABo4HDMIHAMAkGA1Ud +EwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMB4GA1UdEQQX +MBWCA2JveIIDYm94gglsb2NhbGhvc3QwHQYDVR0OBBYEFByPHUXtfeFgwETKOqQF +6aZtP/18MB8GA1UdIwQYMBaAFOebi8sSuBv4EBdPRxYIyAJYa2kEMDEGA1UdHwQq +MCgwJqAkoCKGIGh0dHA6Ly9jcmwtc2VydmVyOjgwMDAvYmFzaWMuY3JsMA0GCSqG +SIb3DQEBCwUAA4IBAQBJPSwIwNZm69R0SIB7SyLYSkRbCZU34RvR8Qzxcl73mVa/ +6i3+ZlJify+Ma5l2b1JSw4kUrelosy7UFcK5IkW5Fa0aTTsK6/qkM+WspQGlJVZF +DZFBef1ZjgmWMZBHwhapn17EL+F/F7W71e0fOw1LGPfzP3fxCftmNrEcoRR+2s6V +Yk/f0yZH4zqOA5xq4P3wLuZN+O/rRY947z8Sq+lzwXwWyrewA6OivUqt1uYcCdkQ +2lTzXRQsOQi+OJ+tHl2MeuXasm/37dusuoH9EtBioN8cuiT3zwv13QKBzGmxPojM +gk18aH4eLTNQ+CgnqgNVC8vTi0b+R0Gs5+xhOWFh +-----END CERTIFICATE----- diff --git a/services/cert/server_box_key.pem b/services/cert/server_box_key.pem new file mode 100755 index 0000000..fa31451 --- /dev/null +++ b/services/cert/server_box_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDATaV4+GBHTuRe +6NRIGq+RS9mZST2e4mKpQgjGquLGhffFMtxoRkh4KW4LA5RF2cAhIkXFBCZV7VT7 +o0U45cKwu4BYL/vPXU49fpvi3iilodZxSZ4RUmO7HEiXMP7XgxZlz5RgAJAx0NIV +WfhtLevgviH8IJ5nPLc4981b+aiCcZDBNAeuVZ/qEUFe0HSK1kq69Bvc0W90QGpV +TJY2/FznBH/UPfEnVZETD7AzOjMbfB0N95tMD3sXKZqVgqavzaRe70Enrb8xVf8r +HlCcbW7uFICkX1IXQParolYxqKfTubiTXPP0n74Ld4JN5h3Pj7xBaLCW7tOPBTDJ +DoEFyFSfAgMBAAECggEAPnsb+AWPVqvp5HJ2wqS1BZrk7PqQhGae0sqrSk1smD8f +5VkkptarJiuj3v0/OEDLPZjCnYQ/Jm0Rzx7Z12ZDjyOkzEJu6Q2yZaWU5a3z+/js +0WsvagZgdAN5DHK1YnVrmhHLbjPQpfbow1ozrtmcSa3NIfvBK97c0ywkc61W1Gac +cu+U7GU9uAhw8RCCXPVceWDHNNMBkYeZ2yUif+TYv2gIznDxtt6/OfAxXRDoxYvj +VjMZeiheMh9lCuINl4IUrlHtUrjtBwvsX6tQX2s0MnpopD/266lsyOcK+3kQIsWm +DungP44p+JGjhVDQy217fqgdVH2+FqJoEbv/2MQ1JQKBgQDloaF8De0Oqr+SQWhu +VAy2mVDE02nOwhH/qSkIV56UnYRyRwYVH5GfHa0V7WMkK2IrjdTJcBVIXtpiEf0Q +5Hy8HtzTgrFBmq5yHOD5IUx3R0X0GjZmPn+e56nGj7eGpEISIWpYijp0Wnk29t10 +6rnA52KRZpb2li7oxp6b5DXuvQKBgQDWYrMMv3p/WsnXu45lEMUNkTireBrdnQOF +n7CTZYCtAtpYUELpii9182V8ANDf78luMLdrzbuLfmuXfN3CEO4RgzQ70+IDnv+o +kWDTUkgWSFhd67VufLOC+3ZudNnmpOMge/aSkuykbAgAWDCghfm1HWY/Wodz/Mrg +I8+CM4PEiwKBgQCo86oX1zsyzmiTGHLYshiEhPtLRe2UhW+utmyNScJQwDCB0EXm +ZsrC0pfWCUong5AAUaNc7o2KKNfuziNvOV2ZH/AD4yW+CiwNo7fXNSvImvUaK+sY +gSVu6i4c/QLgGpzOMC3JpTJrB2ImEa5Q5p7zEouQRXYPSeKLvA9Yzajs1QKBgF+H +fYd5r0kemIB0B+CLF3OTOXBWxYno1E/vt4wGl88ATXE62oYcWEez3I3kAy82N0jN +ln4IH8Dp5WGMd3hpeNv+3gCmyriYWg2wMjgYGx0qwY2gYalJEeiUytIvaYV4BelM +s7Pemmot5WbZ5VkyOfH1lsE2QtNxWqmD64x2Dgo/AoGAD6W3l8s3SB2gf+YCDyTk +FFz5vGjEsLNCRhFRCrSEh2DpOs5y+4hDkZO0fAP2i8IEPjciBGFekoqfOLELr6a7 +rVwEiThilLTSxIiVP9VoX3/KK0K8RYEoAwXCl8Aw/1nPXzhxhGr1Z9ZdQ8DqheUB +1uuaoqudqtEa1MrE1X1Pk0I= +-----END PRIVATE KEY----- diff --git a/services/docker-compose.yaml b/services/docker-compose.yaml new file mode 100644 index 0000000..231ab1d --- /dev/null +++ b/services/docker-compose.yaml @@ -0,0 +1,47 @@ +version: "3" +services: + rabbitmq_stream_3_13: + container_name: rabbitmq_stream + image: rabbitmq:3.13-rc-management + restart: always + hostname: rabbitmq_stream + volumes: + - ./enabled_plugins:/etc/rabbitmq/enabled_plugins + - ./rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf + - ./cert:/etc/rabbitmq/cert + ports: + - 5553:5552 + - 5551:5551 + environment: + - RABBITMQ_DEFAULT_USER=guest + - RABBITMQ_DEFAULT_PASS=guest + rabbitmq_stream_3_12: + container_name: rabbitmq_stream + image: rabbitmq:3.12-management + restart: always + hostname: rabbitmq_stream + volumes: + - ./enabled_plugins:/etc/rabbitmq/enabled_plugins + - ./rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf + - ./cert:/etc/rabbitmq/cert + ports: + - 5553:5552 + - 5551:5551 + environment: + - RABBITMQ_DEFAULT_USER=guest + - RABBITMQ_DEFAULT_PASS=guest + rabbitmq_stream_3_11: + container_name: rabbitmq_stream + image: rabbitmq:3.11-management + restart: always + hostname: rabbitmq_stream + volumes: + - ./enabled_plugins:/etc/rabbitmq/enabled_plugins + - ./rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf + - ./cert:/etc/rabbitmq/cert + ports: + - 5553:5552 + - 5551:5551 + environment: + - RABBITMQ_DEFAULT_USER=guest + - RABBITMQ_DEFAULT_PASS=guest \ No newline at end of file diff --git a/services/enabled_plugins b/services/enabled_plugins new file mode 100644 index 0000000..4ffafe3 --- /dev/null +++ b/services/enabled_plugins @@ -0,0 +1 @@ +[rabbitmq_management, rabbitmq_stream]. \ No newline at end of file diff --git a/services/rabbitmq.conf b/services/rabbitmq.conf new file mode 100644 index 0000000..dd6eaee --- /dev/null +++ b/services/rabbitmq.conf @@ -0,0 +1,10 @@ +log.file.level = debug + +stream.listeners.tcp.1 = 5552 +stream.listeners.ssl.1 = 5551 + +ssl_options.cacertfile = /etc/rabbitmq/cert/ca_certificate.pem +ssl_options.certfile = /etc/rabbitmq/cert/server_box_certificate.pem +ssl_options.keyfile = /etc/rabbitmq/cert/server_box_key.pem +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = true \ No newline at end of file diff --git a/test/connection_test.exs b/test/connection_test.exs index 9756273..e301066 100644 --- a/test/connection_test.exs +++ b/test/connection_test.exs @@ -3,26 +3,58 @@ defmodule RabbitMQStreamTest.Connection do alias RabbitMQStream.Connection import ExUnit.CaptureLog + @moduletag :v3_11 + @moduletag :v3_12 + @moduletag :v3_13 + defmodule SupervisedConnection do use RabbitMQStream.Connection end + defmodule SupervisedSSLConnection do + use RabbitMQStream.Connection, + port: 5551, + transport: :ssl, + ssl_opts: [ + keyfile: "services/cert/client_box_key.pem", + certfile: "services/cert/client_box_certificate.pem", + cacertfile: "services/cert/ca_certificate.pem", + verify: :verify_peer + ] + end + test "should open and close the connection" do - {:ok, _} = SupervisedConnection.start_link(host: "localhost", vhost: "/", lazy: true) + {:ok, conn} = SupervisedConnection.start_link(host: "localhost", vhost: "/", lazy: true) - assert %Connection{state: :closed} = SupervisedConnection.get_state() + assert %Connection{state: :closed} = :sys.get_state(conn) assert :ok = SupervisedConnection.connect() - assert %Connection{state: :open} = SupervisedConnection.get_state() + assert %Connection{state: :open} = :sys.get_state(conn) assert :ok = SupervisedConnection.close() - assert %Connection{state: :closed} = SupervisedConnection.get_state() + assert %Connection{state: :closed} = :sys.get_state(conn) assert :ok = SupervisedConnection.close() end + test "should open and close a ssl connection" do + {:ok, conn} = SupervisedSSLConnection.start_link(host: "localhost", vhost: "/", lazy: true) + + assert %Connection{state: :closed} = :sys.get_state(conn) + + assert :ok = SupervisedSSLConnection.connect() + + assert %Connection{state: :open} = :sys.get_state(conn) + + assert :ok = SupervisedSSLConnection.close() + + assert %Connection{state: :closed} = :sys.get_state(conn) + + assert :ok = SupervisedSSLConnection.close() + end + test "should correctly answer to parallel `connect` requests" do {:ok, _} = SupervisedConnection.start_link(host: "localhost", vhost: "/", lazy: true) @@ -94,36 +126,46 @@ defmodule RabbitMQStreamTest.Connection do end @stream "test-store-05" - test "should declare and delete a publisher" do + test "should declare and delete a producer" do {:ok, _} = SupervisedConnection.start_link(host: "localhost", vhost: "/") :ok = SupervisedConnection.connect() SupervisedConnection.delete_stream(@stream) :ok = SupervisedConnection.create_stream(@stream) - # The publisherId sequence should always start at 1 - assert {:ok, 1} = SupervisedConnection.declare_publisher(@stream, "publisher-01") + # The producerId sequence should always start at 1 + assert {:ok, 1} = SupervisedConnection.declare_producer(@stream, "producer-01") - assert :ok = SupervisedConnection.delete_publisher(1) + assert :ok = SupervisedConnection.delete_producer(1) :ok = SupervisedConnection.delete_stream(@stream) SupervisedConnection.close() end @stream "test-store-06" - @publisher "publisher-02" - test "should query publisher sequence" do + @producer "producer-02" + test "should query producer sequence" do {:ok, _} = SupervisedConnection.start_link(host: "localhost", vhost: "/") :ok = SupervisedConnection.connect() SupervisedConnection.delete_stream(@stream) :ok = SupervisedConnection.create_stream(@stream) - {:ok, _} = SupervisedConnection.declare_publisher(@stream, @publisher) + {:ok, _} = SupervisedConnection.declare_producer(@stream, @producer) - # Should be 0 since the publisher was just declared - assert {:ok, 0} = SupervisedConnection.query_publisher_sequence(@stream, @publisher) + # Should be 0 since the producer was just declared + assert {:ok, 0} = SupervisedConnection.query_producer_sequence(@stream, @producer) - :ok = SupervisedConnection.delete_publisher(1) + :ok = SupervisedConnection.delete_producer(1) :ok = SupervisedConnection.delete_stream(@stream) SupervisedConnection.close() end + + @stream "test-store-07" + test "should get stream stats" do + {:ok, _} = SupervisedConnection.start_link(host: "localhost", vhost: "/") + :ok = SupervisedConnection.connect() + SupervisedConnection.delete_stream(@stream) + :ok = SupervisedConnection.create_stream(@stream) + assert {:ok, _data} = SupervisedConnection.stream_stats(@stream) + assert {:error, :stream_does_not_exist} = SupervisedConnection.stream_stats("#{@stream}-NON-EXISTENT") + end end diff --git a/test/subscriber_test.exs b/test/consumer_test.exs similarity index 52% rename from test/subscriber_test.exs rename to test/consumer_test.exs index cb8b925..d5c70ca 100644 --- a/test/subscriber_test.exs +++ b/test/consumer_test.exs @@ -1,26 +1,44 @@ -defmodule RabbitMQStreamTest.Subscriber do +defmodule RabbitMQStreamTest.Consumer do use ExUnit.Case, async: false alias RabbitMQStream.OsirisChunk - require Logger + + @moduletag :v3_11 + @moduletag :v3_12 + @moduletag :v3_13 defmodule SupervisedConnection do use RabbitMQStream.Connection end - defmodule SupervisorPublisher do - use RabbitMQStream.Publisher, - connection: RabbitMQStreamTest.Subscriber.SupervisedConnection + defmodule SupervisorProducer do + use RabbitMQStream.Producer, + connection: SupervisedConnection + + @impl true + def before_start(_opts, state) do + RabbitMQStream.Connection.create_stream(state.connection, state.stream_name) + + state + end + end + + defmodule SupervisorProducer2 do + use RabbitMQStream.Producer, + connection: SupervisedConnection, + serializer: Jason + @impl true def before_start(_opts, state) do - state.connection.create_stream(state.stream_name) + RabbitMQStream.Connection.create_stream(state.connection, state.stream_name) state end end - defmodule Subscriber do - use RabbitMQStream.Subscriber, - connection: RabbitMQStreamTest.Subscriber.SupervisedConnection + defmodule Consumer do + use RabbitMQStream.Consumer, + connection: SupervisedConnection, + serializer: Jason @impl true def handle_chunk(%OsirisChunk{data_entries: entries}, %{private: parent}) do @@ -37,45 +55,45 @@ defmodule RabbitMQStreamTest.Subscriber do :ok end - @stream "subscriber-test-stream-01" + @stream "consumer-test-stream-01" @reference_name "reference-01" test "should publish and receive a message" do - {:ok, _publisher} = SupervisorPublisher.start_link(reference_name: @reference_name, stream_name: @stream) + {:ok, _producer} = SupervisorProducer.start_link(reference_name: @reference_name, stream_name: @stream) assert {:ok, subscription_id} = SupervisedConnection.subscribe(@stream, self(), :next, 999) - message = inspect(%{message: "Hello, world2!"}) + message = Jason.encode!(%{message: "Hello, world2!"}) - SupervisorPublisher.publish(message) + SupervisorProducer.publish(message) assert_receive {:chunk, %OsirisChunk{data_entries: [^message]}}, 500 assert :ok = SupervisedConnection.unsubscribe(subscription_id) - SupervisorPublisher.publish(message) + SupervisorProducer.publish(message) refute_receive {:chunk, %OsirisChunk{}}, 500 SupervisedConnection.delete_stream(@stream) end - @stream "subscriber-test-stream-02" + @stream "consumer-test-stream-02" @reference_name "reference-02" - test "should credit a subscriber" do - {:ok, _publisher} = SupervisorPublisher.start_link(reference_name: @reference_name, stream_name: @stream) + test "should credit a consumer" do + {:ok, _producer} = SupervisorProducer.start_link(reference_name: @reference_name, stream_name: @stream) - # We ensure the stream exists before subscribing + # We ensure the stream exists before consuming SupervisedConnection.create_stream(@stream) assert {:ok, subscription_id} = SupervisedConnection.subscribe(@stream, self(), :next, 1) - message = inspect(%{message: "Hello, world1!"}) + message = Jason.encode!(%{message: "Hello, world!"}) - SupervisorPublisher.publish(message) + SupervisorProducer.publish(message) assert_receive {:chunk, %OsirisChunk{data_entries: [^message]}}, 500 - message = inspect(%{message: "Hello, world2!"}) + message = Jason.encode!(%{message: "Hello, world2!"}) - SupervisorPublisher.publish(message) + SupervisorProducer.publish(message) refute_receive {:chunk, %OsirisChunk{}}, 500 @@ -85,35 +103,35 @@ defmodule RabbitMQStreamTest.Subscriber do SupervisedConnection.delete_stream(@stream) end - @stream "subscriber-test-stream-10" + @stream "consumer-test-stream-10" @reference_name "reference-10" - test "a message should be received by a persistent subscriber" do + test "a message should be received by a persistent consumer" do SupervisedConnection.delete_stream(@stream) - {:ok, _publisher} = - SupervisorPublisher.start_link(reference_name: @reference_name, stream_name: @stream) + {:ok, _producer} = + SupervisorProducer2.start_link(reference_name: @reference_name, stream_name: @stream) {:ok, _subscriber} = - Subscriber.start_link( + Consumer.start_link( initial_offset: :next, stream_name: @stream, private: self(), offset_tracking: [count: [store_after: 1]] ) - message1 = "Subscriber Test: 1" - message2 = "Subscriber Test: 2" + message1 = %{"message" => "Consumer Test: 1"} + message2 = %{"message" => "Consumer Test: 2"} - SupervisorPublisher.publish(message1) + SupervisorProducer2.publish(message1) assert_receive {:handle_chunk, [^message1]}, 500 - SupervisorPublisher.publish(message2) + SupervisorProducer2.publish(message2) assert_receive {:handle_chunk, [^message2]}, 500 - :ok = GenServer.stop(Subscriber, :normal) + :ok = GenServer.stop(Consumer, :normal) {:ok, _subscriber} = - Subscriber.start_link( + Consumer.start_link( initial_offset: :next, stream_name: @stream, private: self(), diff --git a/test/filter_value_consumer.exs b/test/filter_value_consumer.exs new file mode 100644 index 0000000..e273f15 --- /dev/null +++ b/test/filter_value_consumer.exs @@ -0,0 +1,96 @@ +defmodule RabbitMQStreamTest.Consumer.FilterValue do + use ExUnit.Case, async: false + + @moduletag :v3_13 + + defmodule Conn1 do + use RabbitMQStream.Connection + end + + defmodule Producer do + use RabbitMQStream.Producer, + connection: Conn1, + serializer: Jason + + @impl true + def filter_value(message) do + message["key"] + end + end + + defmodule Subs1 do + use RabbitMQStream.Consumer, + connection: Conn1, + initial_offset: :next, + stream_name: "filter-value-01", + offset_tracking: [count: [store_after: 1]], + properties: [filter: ["Subs1"]], + serializer: Jason + + @impl true + def handle_chunk(%{data_entries: [entry]}, %{private: parent}) do + send(parent, {__MODULE__, entry}) + + :ok + end + end + + defmodule Subs2 do + use RabbitMQStream.Consumer, + connection: Conn1, + initial_offset: :next, + stream_name: "filter-value-01", + offset_tracking: [count: [store_after: 1]], + properties: [filter: ["Subs2"]], + serializer: Jason + + @impl true + def handle_chunk(%{data_entries: [entry]}, %{private: parent}) do + send(parent, {__MODULE__, entry}) + + :ok + end + end + + defmodule Subs3 do + use RabbitMQStream.Consumer, + connection: Conn1, + initial_offset: :next, + stream_name: "filter-value-01", + offset_tracking: [count: [store_after: 1]], + properties: [match_unfiltered: true], + serializer: Jason + + @impl true + def handle_chunk(%{data_entries: [entry]}, %{private: parent}) do + send(parent, {__MODULE__, entry}) + + :ok + end + end + + test "should receive only the filtered messages" do + {:ok, _} = Conn1.start_link() + :ok = Conn1.connect() + Conn1.delete_stream("filter-value-01") + :ok = Conn1.create_stream("filter-value-01") + {:ok, _} = Producer.start_link(stream_name: "filter-value-01") + + {:ok, _} = Subs1.start_link(private: self()) + {:ok, _} = Subs2.start_link(private: self()) + {:ok, _} = Subs3.start_link(private: self()) + + message = %{"key" => "Subs1", "value" => "1"} + :ok = Producer.publish(message) + + assert_receive {Subs1, ^message}, 500 + + message = %{"key" => "Subs2", "value" => "2"} + :ok = Producer.publish(message) + assert_receive {Subs2, ^message}, 500 + + message = %{"key" => "NO-FILTER-APPLIED", "value" => "3"} + :ok = Producer.publish(message) + assert_receive {Subs3, ^message}, 500 + end +end diff --git a/test/producer_test.exs b/test/producer_test.exs new file mode 100644 index 0000000..2226e4d --- /dev/null +++ b/test/producer_test.exs @@ -0,0 +1,78 @@ +defmodule RabbitMQStreamTest.Producer do + use ExUnit.Case, async: false + + @moduletag :v3_11 + @moduletag :v3_12 + @moduletag :v3_13 + + defmodule SupervisedConnection do + use RabbitMQStream.Connection + end + + defmodule SupervisorProducer do + use RabbitMQStream.Producer, + connection: RabbitMQStreamTest.Producer.SupervisedConnection + + @impl true + def before_start(_opts, state) do + RabbitMQStream.Connection.create_stream(state.connection, state.stream_name) + + state + end + end + + setup do + {:ok, _conn} = SupervisedConnection.start_link(host: "localhost", vhost: "/") + :ok = SupervisedConnection.connect() + + :ok + end + + @stream "producer-test-01" + @reference_name "producer-test-reference-01" + test "should declare itself and its stream" do + assert {:ok, _} = + SupervisorProducer.start_link(reference_name: @reference_name, stream_name: @stream) + + SupervisedConnection.delete_stream(@stream) + end + + @stream "producer-test-02" + @reference_name "producer-test-reference-02" + test "should query its sequence when declaring" do + {:ok, _} = + SupervisorProducer.start_link( + reference_name: @reference_name, + stream_name: @stream + ) + + assert %{sequence: 1} = :sys.get_state(Process.whereis(SupervisorProducer)) + SupervisedConnection.delete_stream(@stream) + end + + @stream "producer-test-03" + @reference_name "producer-test-reference-03" + test "should publish a message" do + {:ok, _} = + SupervisorProducer.start_link( + reference_name: @reference_name, + stream_name: @stream + ) + + %{sequence: sequence} = :sys.get_state(Process.whereis(SupervisorProducer)) + + SupervisorProducer.publish(inspect(%{message: "Hello, world!"})) + + sequence = sequence + 1 + + assert %{sequence: ^sequence} = :sys.get_state(Process.whereis(SupervisorProducer)) + + SupervisorProducer.publish(inspect(%{message: "Hello, world2!"})) + + sequence = sequence + 1 + + assert %{sequence: ^sequence} = :sys.get_state(Process.whereis(SupervisorProducer)) + + SupervisedConnection.delete_stream(@stream) + end +end diff --git a/test/publisher_test.exs b/test/publisher_test.exs deleted file mode 100644 index 2dd5a27..0000000 --- a/test/publisher_test.exs +++ /dev/null @@ -1,105 +0,0 @@ -defmodule RabbitMQStreamTest.Publisher do - use ExUnit.Case, async: false - - alias RabbitMQStream.Connection - - defmodule SupervisedConnection do - use RabbitMQStream.Connection - end - - defmodule SupervisorPublisher do - use RabbitMQStream.Publisher, - connection: RabbitMQStreamTest.Publisher.SupervisedConnection - - def before_start(_opts, state) do - state.connection.create_stream(state.stream_name) - - state - end - end - - setup do - {:ok, _conn} = SupervisedConnection.start_link(host: "localhost", vhost: "/") - :ok = SupervisedConnection.connect() - - :ok - end - - @stream "publisher-test-01" - @reference_name "publisher-test-reference-01" - test "should declare itself and its stream" do - assert {:ok, _} = - SupervisorPublisher.start_link( - reference_name: @reference_name, - stream_name: @stream - ) - - SupervisedConnection.delete_stream(@stream) - end - - @stream "publisher-test-02" - @reference_name "publisher-test-reference-02" - test "should query its sequence when declaring" do - {:ok, _} = - SupervisorPublisher.start_link( - reference_name: @reference_name, - stream_name: @stream - ) - - assert %{sequence: 1} = SupervisorPublisher.get_state() - SupervisedConnection.delete_stream(@stream) - end - - @stream "publisher-test-03" - @reference_name "publisher-test-reference-03" - test "should publish a message" do - {:ok, _} = - SupervisorPublisher.start_link( - reference_name: @reference_name, - stream_name: @stream - ) - - %{sequence: sequence} = SupervisorPublisher.get_state() - - SupervisorPublisher.publish(inspect(%{message: "Hello, world!"})) - - sequence = sequence + 1 - - assert %{sequence: ^sequence} = SupervisorPublisher.get_state() - - SupervisorPublisher.publish(inspect(%{message: "Hello, world2!"})) - - sequence = sequence + 1 - - assert %{sequence: ^sequence} = SupervisorPublisher.get_state() - - SupervisedConnection.delete_stream(@stream) - end - - @stream "publisher-test-04" - @reference_name "publisher-test-reference-04" - test "should keep track of sequence across startups" do - {:ok, _} = - SupervisorPublisher.start_link( - reference_name: @reference_name, - stream_name: @stream - ) - - SupervisorPublisher.publish(inspect(%{message: "Hello, world!"})) - SupervisorPublisher.publish(inspect(%{message: "Hello, world2!"})) - - %{sequence: sequence} = SupervisorPublisher.get_state() - - assert :ok = SupervisorPublisher.stop() - - {:ok, _} = - SupervisorPublisher.start_link( - reference_name: @reference_name, - stream_name: @stream - ) - - assert %{sequence: ^sequence} = SupervisorPublisher.get_state() - - SupervisedConnection.delete_stream(@stream) - end -end diff --git a/test/single_active_consumer_test.exs b/test/single_active_consumer_test.exs new file mode 100644 index 0000000..d55108d --- /dev/null +++ b/test/single_active_consumer_test.exs @@ -0,0 +1,141 @@ +defmodule RabbitMQStreamTest.Consumer.SingleActiveConsumer do + use ExUnit.Case, async: false + require Logger + + @moduletag :v3_11 + @moduletag :v3_12 + @moduletag :v3_13 + + defmodule Conn1 do + use RabbitMQStream.Connection + end + + defmodule Conn2 do + use RabbitMQStream.Connection + end + + defmodule Conn3 do + use RabbitMQStream.Connection + end + + defmodule Conn4 do + use RabbitMQStream.Connection + end + + defmodule Producer do + use RabbitMQStream.Producer, + connection: Conn4 + + @impl true + def before_start(_opts, state) do + RabbitMQStream.Connection.create_stream(state.connection, state.stream_name) + + state + end + end + + defmodule Subs1 do + use RabbitMQStream.Consumer, + connection: Conn1, + initial_offset: :next, + stream_name: "super-stream-test-01", + offset_tracking: [count: [store_after: 1]], + properties: [single_active_consumer: "group-1"] + + @impl true + def handle_update(_, true) do + {:ok, :last} + end + + @impl true + def handle_chunk(%{data_entries: [entry]}, %{private: parent, connection: conn}) do + send(parent, {{conn, __MODULE__}, entry}) + + :ok + end + end + + defmodule Subs2 do + use RabbitMQStream.Consumer, + connection: Conn2, + initial_offset: :next, + stream_name: "super-stream-test-01", + offset_tracking: [count: [store_after: 1]], + properties: [single_active_consumer: "group-1"] + + @impl true + def handle_update(_, true) do + {:ok, :last} + end + + @impl true + def handle_chunk(%{data_entries: [entry]}, %{private: parent, connection: conn}) do + send(parent, {{conn, __MODULE__}, entry}) + + :ok + end + end + + defmodule Subs3 do + use RabbitMQStream.Consumer, + connection: Conn3, + initial_offset: :next, + stream_name: "super-stream-test-01", + offset_tracking: [count: [store_after: 1]], + properties: [single_active_consumer: "group-1"] + + @impl true + def handle_update(_, true) do + {:ok, :last} + end + + @impl true + def handle_chunk(%{data_entries: [entry]}, %{private: parent, connection: conn}) do + send(parent, {{conn, __MODULE__}, entry}) + + :ok + end + end + + test "should always have exactly 1 active consumer" do + {:ok, _} = Conn1.start_link(name: :conn1) + {:ok, _} = Conn2.start_link(name: :conn2) + {:ok, _} = Conn3.start_link(name: :conn2) + {:ok, _} = Conn4.start_link(name: :conn2) + :ok = Conn1.connect() + :ok = Conn2.connect() + :ok = Conn3.connect() + :ok = Conn4.connect() + + Conn4.delete_stream("super-stream-test-01") + + {:ok, _} = Producer.start_link(stream_name: "super-stream-test-01") + + {:ok, _} = Subs1.start_link(private: self()) + {:ok, _} = Subs2.start_link(private: self()) + {:ok, _} = Subs3.start_link(private: self()) + + :ok = Producer.publish("1") + + assert_receive {{_conn1, sub1}, "1"}, 500 + + :ok = Producer.publish("2") + + assert_receive {{conn1, ^sub1}, "2"}, 500 + + # When calling .stop, the consumer send a 'unsubscribe' request to the connection before closing. + # So we must first close the consumer, then the connection. + :ok = GenServer.stop(sub1) + :ok = GenServer.stop(conn1) + + # At this point, the newly selected consumer should have received + # a 'consumer_update' request from the server and informed its current offset. + # We are setting the defaults offset to ':last' in the `handle_update` callback above. + # But a more realistic scenario would be to fetch the offset from the stream itself. + :ok = Producer.publish("3") + + assert_receive {{_conn, sub2}, "3"}, 500 + + assert sub2 != sub1 + end +end diff --git a/test/super_stream_test.exs b/test/super_stream_test.exs new file mode 100644 index 0000000..3680b24 --- /dev/null +++ b/test/super_stream_test.exs @@ -0,0 +1,163 @@ +defmodule RabbitMQStreamTest.SuperStream do + use ExUnit.Case, async: false + alias RabbitMQStream.OsirisChunk + require Logger + + defmodule SuperConsumer1 do + use RabbitMQStream.SuperConsumer, + initial_offset: :next, + partitions: 3 + + @impl true + def handle_chunk(%OsirisChunk{}, %{private: parent}) do + send(parent, __MODULE__) + + :ok + end + + @impl true + def handle_update(state, _) do + {:ok, state.last_offset || state.initial_offset} + end + end + + defmodule SuperConsumer2 do + use RabbitMQStream.SuperConsumer, + initial_offset: :next, + partitions: 3 + + @impl true + def handle_chunk(%OsirisChunk{}, %{private: parent}) do + send(parent, __MODULE__) + + :ok + end + + @impl true + def handle_update(state, _) do + {:ok, state.last_offset || state.initial_offset} + end + end + + defmodule SuperConsumer3 do + use RabbitMQStream.SuperConsumer, + initial_offset: :next, + partitions: 3 + + @impl true + def handle_chunk(%OsirisChunk{}, %{private: parent}) do + send(parent, __MODULE__) + + :ok + end + + @impl true + def handle_update(state, _) do + {:ok, state.last_offset || state.initial_offset} + end + end + + defmodule SuperProducer do + use RabbitMQStream.SuperProducer, + partitions: 3 + end + + setup do + {:ok, conn} = RabbitMQStream.Connection.start_link(host: "localhost", vhost: "/") + :ok = RabbitMQStream.Connection.connect(conn) + + [conn: conn] + end + + @tag :v3_13 + test "should create and delete a super_stream", %{conn: conn} do + RabbitMQStream.Connection.delete_super_stream(conn, "test") + + :ok = + RabbitMQStream.Connection.create_super_stream(conn, "test", + "test-0": 0, + "test-1": 1, + "test-2": 2 + ) + + {:ok, %{streams: ["test-0"]}} = RabbitMQStream.Connection.route(conn, "0", "test") + {:ok, %{streams: ["test-1"]}} = RabbitMQStream.Connection.route(conn, "1", "test") + {:ok, %{streams: ["test-2"]}} = RabbitMQStream.Connection.route(conn, "2", "test") + + {:ok, %{streams: streams}} = RabbitMQStream.Connection.partitions(conn, "test") + + assert Enum.all?(streams, fn stream -> stream in ["test-0", "test-1", "test-2"] end) + + :ok = RabbitMQStream.Connection.delete_super_stream(conn, "test") + end + + @tag :v3_11 + @tag :v3_12 + @tag :v3_13 + test "should create super streams" do + {:ok, conn} = RabbitMQStream.Connection.start_link(host: "localhost", vhost: "/") + :ok = RabbitMQStream.Connection.connect(conn) + + {:ok, %{streams: streams}} = + RabbitMQStream.Connection.query_metadata(conn, ["invoices-0", "invoices-1", "invoices-2"]) + + unless Enum.all?(streams, &(&1.code == :ok)) do + raise "SuperStream streams were not found. Please ensure you've created the \"invoices\" SuperStream with 3 partitions using RabbitMQ CLI or Management UI before running this test." + end + + {:ok, _} = + SuperConsumer1.start_link( + connection: conn, + super_stream: "invoices", + private: self() + ) + + {:ok, conn} = RabbitMQStream.Connection.start_link(host: "localhost", vhost: "/") + :ok = RabbitMQStream.Connection.connect(conn) + + {:ok, _} = + SuperConsumer2.start_link( + connection: conn, + super_stream: "invoices", + private: self() + ) + + {:ok, conn} = RabbitMQStream.Connection.start_link(host: "localhost", vhost: "/") + :ok = RabbitMQStream.Connection.connect(conn) + + {:ok, _} = + SuperConsumer3.start_link( + connection: conn, + super_stream: "invoices", + private: self() + ) + + {:ok, conn} = RabbitMQStream.Connection.start_link(host: "localhost", vhost: "/") + :ok = RabbitMQStream.Connection.connect(conn) + + {:ok, _} = + SuperProducer.start_link( + connection: conn, + super_stream: "invoices" + ) + + # We wait a bit to guarantee that the consumers are ready + Process.sleep(500) + + :ok = SuperProducer.publish("1") + :ok = SuperProducer.publish("12") + :ok = SuperProducer.publish("123") + + msgs = + for _ <- 1..3 do + receive do + msg -> msg + end + end + + # Process.sleep(60_000) + assert SuperConsumer1 in msgs + assert SuperConsumer2 in msgs + assert SuperConsumer3 in msgs + end +end diff --git a/test/supervision_test.exs b/test/supervision_test.exs deleted file mode 100644 index 34baea5..0000000 --- a/test/supervision_test.exs +++ /dev/null @@ -1,61 +0,0 @@ -defmodule RabbitMQStreamTest.Supervision do - use ExUnit.Case, async: false - - defmodule SupervisedConnection do - use RabbitMQStream.Connection - end - - defmodule SupervisorPublisher do - use RabbitMQStream.Publisher, - connection: RabbitMQStreamTest.Supervision.SupervisedConnection - - def before_start(_opts, state) do - state.connection.create_stream(state.stream_name) - - state - end - end - - defmodule MySupervisor do - use Supervisor - - def start_link do - Supervisor.start_link(__MODULE__, :ok, name: __MODULE__) - end - - @stream "stream-08" - @reference_name "reference-08" - - def init(:ok) do - children = [ - RabbitMQStreamTest.Supervision.SupervisedConnection, - {RabbitMQStreamTest.Supervision.SupervisorPublisher, reference_name: @reference_name, stream_name: @stream} - ] - - Supervisor.init(children, strategy: :one_for_all) - end - end - - @stream "stream-01" - @reference_name "reference-01" - - test "should start itself and publish a message" do - {:ok, _conn} = SupervisedConnection.start_link(host: "localhost", vhost: "/") - :ok = SupervisedConnection.connect() - assert {:ok, _publisher} = SupervisorPublisher.start_link(reference_name: @reference_name, stream_name: @stream) - - %{sequence: sequence} = SupervisorPublisher.get_state() - - assert :ok = SupervisorPublisher.publish("Hello, world!") - - sequence = sequence + 1 - - assert %{sequence: ^sequence} = SupervisorPublisher.get_state() - SupervisedConnection.delete_stream(@stream) - end - - test "should start itself and publish a message via supervisor" do - {:ok, _supervisor} = MySupervisor.start_link() - assert :ok = SupervisorPublisher.publish("Hello, world!") - end -end