From ed29416643f73a59a56c902a4f3e892eccfff880 Mon Sep 17 00:00:00 2001 From: Henrik Rasmussen Date: Tue, 18 Feb 2025 20:44:53 +0100 Subject: [PATCH 1/5] WIP: Support for authentication Service principal This first draft is to get early feedback on the approach to support Service Principal Aauthentication * In config.ex we determine which type of autentication the user has choosen * The selected method is used via the Auth module which is to replace the SharedKey.sign function everywhere * Introduced an azure_integration test tag that will test against the real azure blob stoage --- lib/azurex/authorization/auth.ex | 25 +++++++++ lib/azurex/authorization/service_principal.ex | 33 ++++++++++++ lib/azurex/authorization/shared_key.ex | 2 +- lib/azurex/blob.ex | 7 +-- lib/azurex/blob/block.ex | 7 +-- lib/azurex/blob/config.ex | 52 +++++++++++++++++++ lib/azurex/blob/container.ex | 7 +-- mix.exs | 3 +- mix.lock | 1 + test/support/azure_setup.ex | 49 +++++++++++++++++ test/test_helper.exs | 24 ++++++--- 11 files changed, 186 insertions(+), 24 deletions(-) create mode 100644 lib/azurex/authorization/auth.ex create mode 100644 lib/azurex/authorization/service_principal.ex create mode 100644 test/support/azure_setup.ex diff --git a/lib/azurex/authorization/auth.ex b/lib/azurex/authorization/auth.ex new file mode 100644 index 0000000..a3120b2 --- /dev/null +++ b/lib/azurex/authorization/auth.ex @@ -0,0 +1,25 @@ +defmodule Azurex.Authorization.Auth do + alias Azurex.Blob.Config + alias Azurex.Authorization.SharedKey + + def authorize_request(request, content_type \\ "") do + case Config.auth_method() do + {:account_key, storage_account_key} -> + SharedKey.sign( + request, + storage_account_name: Config.storage_account_name(), + storage_account_key: storage_account_key, + content_type: content_type + ) + + {:service_principal, client_id, client_secret, tenant} -> + Azurex.Authorization.ServicePrincipal.add_bearer_token( + request, + client_id, + client_secret, + tenant + ) + |> SharedKey.put_standard_headers(content_type, DateTime.utc_now()) + end + end +end diff --git a/lib/azurex/authorization/service_principal.ex b/lib/azurex/authorization/service_principal.ex new file mode 100644 index 0000000..3eeee2a --- /dev/null +++ b/lib/azurex/authorization/service_principal.ex @@ -0,0 +1,33 @@ +defmodule Azurex.Authorization.ServicePrincipal do + def add_bearer_token(%HTTPoison.Request{} = request, client_id, client_secret, tenant_id) do + bearer_token = fetch_bearer_token(client_id, client_secret, tenant_id) + authorization = {"Authorization", "Bearer #{bearer_token}"} + + headers = [authorization | request.headers] + struct(request, headers: headers) + end + + def fetch_bearer_token(client_id, client_secret, tenant_id) do + body = + "grant_type=client_credentials&client_id=#{client_id}&client_secret=#{client_secret}&scope=https://storage.azure.com/.default" + + respone = + %HTTPoison.Request{ + method: :post, + url: "https://login.microsoftonline.com/#{tenant_id}/oauth2/v2.0/token", + body: body, + headers: [ + {"content-type", "application/x-www-form-urlencoded"} + ] + } + |> HTTPoison.request() + + case respone do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + body |> Jason.decode!() |> Map.get("access_token") + + {:error, err} -> + {:error, err} + end + end +end diff --git a/lib/azurex/authorization/shared_key.ex b/lib/azurex/authorization/shared_key.ex index eb4c1ef..1076819 100644 --- a/lib/azurex/authorization/shared_key.ex +++ b/lib/azurex/authorization/shared_key.ex @@ -58,7 +58,7 @@ defmodule Azurex.Authorization.SharedKey do put_signature(request, signature, storage_account_name, storage_account_key) end - defp put_standard_headers(request, content_type, date) do + def put_standard_headers(request, content_type, date) do headers = if content_type, do: [{"content-type", content_type} | request.headers], diff --git a/lib/azurex/blob.ex b/lib/azurex/blob.ex index 3cf1bd9..567df75 100644 --- a/lib/azurex/blob.ex +++ b/lib/azurex/blob.ex @@ -5,6 +5,7 @@ defmodule Azurex.Blob do In the functions below set container as nil to use the one configured in `Azurex.Blob.Config`. """ + alias Azurex.Authorization.Auth alias Azurex.Blob.{Block, Config} alias Azurex.Authorization.SharedKey @@ -106,11 +107,7 @@ defmodule Azurex.Blob do # is not applicable for the put request, so we set it to infinity options: [recv_timeout: :infinity] } - |> SharedKey.sign( - storage_account_name: Config.storage_account_name(), - storage_account_key: Config.storage_account_key(), - content_type: content_type - ) + |> Auth.authorize_request(content_type) |> HTTPoison.request() |> case do {:ok, %{status_code: 201}} -> :ok diff --git a/lib/azurex/blob/block.ex b/lib/azurex/blob/block.ex index a89811e..42ce76d 100644 --- a/lib/azurex/blob/block.ex +++ b/lib/azurex/blob/block.ex @@ -8,6 +8,7 @@ defmodule Azurex.Blob.Block do - [commit a list of blocks as part of a blob](https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list) """ + alias Azurex.Authorization.Auth alias Azurex.Authorization.SharedKey alias Azurex.Blob alias Azurex.Blob.Config @@ -82,11 +83,7 @@ defmodule Azurex.Blob.Block do {"x-ms-blob-content-type", blob_content_type} ] } - |> SharedKey.sign( - storage_account_name: Config.storage_account_name(), - storage_account_key: Config.storage_account_key(), - content_type: content_type - ) + |> Auth.authorize_request(content_type) |> HTTPoison.request() |> case do {:ok, %HTTPoison.Response{status_code: 201}} -> :ok diff --git a/lib/azurex/blob/config.ex b/lib/azurex/blob/config.ex index ec7b298..62a7bc9 100644 --- a/lib/azurex/blob/config.ex +++ b/lib/azurex/blob/config.ex @@ -44,6 +44,58 @@ defmodule Azurex.Blob.Config do end || raise @missing_envs_error_msg end + defp try_account_key_env(nil) do + case Keyword.get(conf(), :storage_account_key) do + nil -> nil + key -> {:account_key, Base.decode64!(key)} + end + end + + defp try_account_key_env(value), do: value + + defp try_account_key_conn_string(nil) do + case get_connection_string_value("AccountKey") do + nil -> nil + key -> {:account_key, Base.decode64!(key)} + end + end + + defp try_account_key_conn_string(value), do: value + + defp try_service_principal(nil) do + {missing_values, values} = + [:storage_client_id, :storage_client_secret, :storage_tenant_id] + |> Enum.map(&Keyword.get(conf(), &1)) + |> Enum.split_with(&is_nil/1) + + case values do + [client_id, client_secret, tenant] -> + {:service_principal, client_id, client_secret, tenant} + + [] -> + nil + + _ -> + raise "Missing values for service principal #{Enum.join(missing_values, ", ")}" + end + end + + defp try_service_principal(value), do: value + + @spec auth_method() :: + {:service_principal, binary(), binary(), binary()} | {:account_key, binary()} + def auth_method do + nil + |> try_account_key_env + |> try_account_key_conn_string + |> try_service_principal || + raise """ + Missing credentials settings. + Either by storage account key: `storage_account_name` and `storage_account_key` or `storage_account_connection_string` + Or by service principal: `storage_client_id`, `storage_client_secret` and `storage_tenant_id` + """ + end + @doc """ Azure storage account access key. Base64 encoded, as provided by azure UI. Required if `storage_account_connection_string` not set. diff --git a/lib/azurex/blob/container.ex b/lib/azurex/blob/container.ex index 0546be7..e103373 100644 --- a/lib/azurex/blob/container.ex +++ b/lib/azurex/blob/container.ex @@ -2,6 +2,7 @@ defmodule Azurex.Blob.Container do @moduledoc """ Implementation of Azure Blob Storage """ + alias Azurex.Authorization.Auth alias Azurex.Blob.Config alias Azurex.Authorization.SharedKey @@ -30,11 +31,7 @@ defmodule Azurex.Blob.Container do params: [restype: "container"], method: :put } - |> SharedKey.sign( - storage_account_name: Config.storage_account_name(), - storage_account_key: Config.storage_account_key(), - content_type: "application/octet-stream" - ) + |> Auth.authorize_request("application/octet-stream") |> HTTPoison.request() |> case do {:ok, %{status_code: 201}} -> {:ok, container} diff --git a/mix.exs b/mix.exs index f28c63f..02b6ecb 100644 --- a/mix.exs +++ b/mix.exs @@ -28,7 +28,8 @@ defmodule Azurex.MixProject do [ {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, - {:httpoison, "~> 1.8 or ~> 2.2"} + {:httpoison, "~> 1.8 or ~> 2.2"}, + {:jason, "~> 1.4.4"} ] end diff --git a/mix.lock b/mix.lock index bfa4265..69689a6 100644 --- a/mix.lock +++ b/mix.lock @@ -10,6 +10,7 @@ "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, "httpoison": {:hex, :httpoison, "2.2.1", "87b7ed6d95db0389f7df02779644171d7319d319178f6680438167d7b69b1f3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "51364e6d2f429d80e14fe4b5f8e39719cacd03eb3f9a9286e61e216feac2d2df"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, + "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.5", "e0ff5a7c708dda34311f7522a8758e23bfcd7d8d8068dc312b5eb41c6fd76eba", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "94d2e986428585a21516d7d7149781480013c56e30c6a233534bedf38867a59a"}, diff --git a/test/support/azure_setup.ex b/test/support/azure_setup.ex new file mode 100644 index 0000000..33fb2aa --- /dev/null +++ b/test/support/azure_setup.ex @@ -0,0 +1,49 @@ +defmodule AzureSetup do + @moduledoc """ + Test setup helper functions for creating containers and blobs in Azurite in + support of integration tests. + """ + + @default_container "test" + @integration_testing_container "integrationtestingcontainer" + @test_blob_name "test_blob" + + def set_env do + Application.put_env(:azurex, Azurex.Blob.Config, + storage_account_name: System.get_env("STORAGE_ACCOUNT_NAME"), + default_container: @default_container, + storage_client_id: System.get_env("STORAGE_CLIENT_ID"), + storage_client_secret: System.get_env("STORAGE_CLIENT_SECRET"), + storage_tenant_id: System.get_env("STORAGE_TENANT_ID") + ) + end + + def create_test_containers do + Enum.each( + [ + @default_container, + @integration_testing_container + ], + &create_test_container(&1) + ) + end + + defp create_test_container(container) do + container + |> Azurex.Blob.Container.create() + |> case do + {:ok, _} -> :ok + {:error, :already_exists} -> :ok + {:error, err} -> raise err + end + end + + def create_test_blob do + Azurex.Blob.put_blob( + @test_blob_name, + "test_blob_content", + "text/plain", + @default_container + ) + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 2044bf4..1386c44 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,13 +1,23 @@ -ExUnit.start(exclude: [:integration]) +ExUnit.start(exclude: [:integration, :azure_integration]) -Application.get_env(:ex_unit, :include) -|> Enum.member?(:integration) -|> case do - true -> +included = Application.get_env(:ex_unit, :include) +is_azure_integration = Enum.member?(included, :azure_integration) +is_integration = Enum.member?(included, :integration) + +case {is_integration, is_azure_integration} do + {false, false} -> + :ok + + {true, false} -> AzuriteSetup.set_env() AzuriteSetup.create_test_containers() AzuriteSetup.create_test_blob() - false -> - :ok + {false, true} -> + AzureSetup.set_env() + AzureSetup.create_test_containers() + AzureSetup.create_test_blob() + + _ -> + raise "Cannot run both integration and azure_integration tests at the same time" end From ceefe62c457ae573aa82fd296ccfd90c43799bd6 Mon Sep 17 00:00:00 2001 From: Henrik Rasmussen Date: Wed, 5 Mar 2025 20:16:20 +0100 Subject: [PATCH 2/5] Spread the use of auth_method to all calls --- lib/azurex/authorization/auth.ex | 3 ++- lib/azurex/blob.ex | 22 ++++------------------ lib/azurex/blob/block.ex | 8 +------- lib/azurex/blob/container.ex | 6 +----- 4 files changed, 8 insertions(+), 31 deletions(-) diff --git a/lib/azurex/authorization/auth.ex b/lib/azurex/authorization/auth.ex index a3120b2..bbe6425 100644 --- a/lib/azurex/authorization/auth.ex +++ b/lib/azurex/authorization/auth.ex @@ -1,6 +1,7 @@ defmodule Azurex.Authorization.Auth do alias Azurex.Blob.Config alias Azurex.Authorization.SharedKey + alias Azurex.Authorization.ServicePrincipal def authorize_request(request, content_type \\ "") do case Config.auth_method() do @@ -13,7 +14,7 @@ defmodule Azurex.Authorization.Auth do ) {:service_principal, client_id, client_secret, tenant} -> - Azurex.Authorization.ServicePrincipal.add_bearer_token( + ServicePrincipal.add_bearer_token( request, client_id, client_secret, diff --git a/lib/azurex/blob.ex b/lib/azurex/blob.ex index 567df75..019c1c3 100644 --- a/lib/azurex/blob.ex +++ b/lib/azurex/blob.ex @@ -7,7 +7,6 @@ defmodule Azurex.Blob do alias Azurex.Authorization.Auth alias Azurex.Blob.{Block, Config} - alias Azurex.Authorization.SharedKey @typep optional_string :: String.t() | nil @@ -16,10 +15,7 @@ defmodule Azurex.Blob do url: Config.api_url() <> "/", params: [comp: "list"] } - |> SharedKey.sign( - storage_account_name: Config.storage_account_name(), - storage_account_key: Config.storage_account_key() - ) + |> Auth.authorize_request() |> HTTPoison.request() |> case do {:ok, %{body: xml, status_code: 200}} -> {:ok, xml} @@ -182,11 +178,7 @@ defmodule Azurex.Blob do url: get_url(container, destination_name), headers: headers } - |> SharedKey.sign( - storage_account_name: Config.storage_account_name(), - storage_account_key: Config.storage_account_key(), - content_type: content_type - ) + |> Auth.authorize_request(content_type) |> HTTPoison.request() |> case do {:ok, %HTTPoison.Response{status_code: 202} = resp} -> {:ok, resp} @@ -216,10 +208,7 @@ defmodule Azurex.Blob do headers: headers, options: options } - |> SharedKey.sign( - storage_account_name: Config.storage_account_name(), - storage_account_key: Config.storage_account_key() - ) + |> Auth.authorize_request() end @doc """ @@ -245,10 +234,7 @@ defmodule Azurex.Blob do restype: "container" ] ++ params } - |> SharedKey.sign( - storage_account_name: Config.storage_account_name(), - storage_account_key: Config.storage_account_key() - ) + |> Auth.authorize_request() |> HTTPoison.request() |> case do {:ok, %{body: xml, status_code: 200}} -> {:ok, xml} diff --git a/lib/azurex/blob/block.ex b/lib/azurex/blob/block.ex index 42ce76d..fb98025 100644 --- a/lib/azurex/blob/block.ex +++ b/lib/azurex/blob/block.ex @@ -9,9 +9,7 @@ defmodule Azurex.Blob.Block do """ alias Azurex.Authorization.Auth - alias Azurex.Authorization.SharedKey alias Azurex.Blob - alias Azurex.Blob.Config @doc """ Creates a block to be committed to a blob. @@ -35,11 +33,7 @@ defmodule Azurex.Blob.Block do {"content-length", byte_size(chunk)} ] } - |> SharedKey.sign( - storage_account_name: Config.storage_account_name(), - storage_account_key: Config.storage_account_key(), - content_type: content_type - ) + |> Auth.authorize_request(content_type) |> HTTPoison.request() |> case do {:ok, %HTTPoison.Response{status_code: 201}} -> {:ok, block_id} diff --git a/lib/azurex/blob/container.ex b/lib/azurex/blob/container.ex index e103373..54e73ac 100644 --- a/lib/azurex/blob/container.ex +++ b/lib/azurex/blob/container.ex @@ -4,7 +4,6 @@ defmodule Azurex.Blob.Container do """ alias Azurex.Authorization.Auth alias Azurex.Blob.Config - alias Azurex.Authorization.SharedKey def head_container(container) do %HTTPoison.Request{ @@ -12,10 +11,7 @@ defmodule Azurex.Blob.Container do params: [restype: "container"], method: :head } - |> SharedKey.sign( - storage_account_name: Config.storage_account_name(), - storage_account_key: Config.storage_account_key() - ) + |> Auth.authorize_request() |> HTTPoison.request() |> case do {:ok, %{status_code: 200, headers: headers}} -> {:ok, headers} From 78734104b90be27ac08c5adac694cdd15987d613 Mon Sep 17 00:00:00 2001 From: Henrik Rasmussen Date: Wed, 5 Mar 2025 20:16:56 +0100 Subject: [PATCH 3/5] Run all test against real blob storage I had to lowercase Azure will return Content-MD5 instead of content-md5 like Azurite --- test/integration/blob_integration_test.exs | 12 ++++-------- test/integration/container_integration_test.exs | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/test/integration/blob_integration_test.exs b/test/integration/blob_integration_test.exs index 06377e7..a48069f 100644 --- a/test/integration/blob_integration_test.exs +++ b/test/integration/blob_integration_test.exs @@ -2,16 +2,11 @@ defmodule Azurex.BlobIntegrationTests do use ExUnit.Case, async: false alias Azurex.Blob - @moduletag integration: true + @moduletag integration: true, azure_integration: true @sample_file_contents "sample file\ncontents\n" @integration_testing_container "integrationtestingcontainer" - setup do - # set integration test env in case another test has overwritten it - AzuriteSetup.set_env() - end - describe "upload and download a blob" do test "using default container" do blob_name = make_blob_name() @@ -97,7 +92,8 @@ defmodule Azurex.BlobIntegrationTests do ) == :ok assert {:ok, headers} = Blob.head_blob(blob_name) - headers = Map.new(headers) + + headers = Enum.map(headers, fn {k, v} -> {String.downcase(k), v} end) |> Map.new() assert headers["content-length"] == byte_size(@sample_file_contents) |> to_string() assert headers["content-type"] == "text/plain" @@ -117,7 +113,7 @@ defmodule Azurex.BlobIntegrationTests do assert {:error, :not_found} = Blob.head_blob(blob_name) assert {:ok, headers} = Blob.head_blob(blob_name, @integration_testing_container) - headers = Map.new(headers) + headers = Enum.map(headers, fn {k, v} -> {String.downcase(k), v} end) |> Map.new() assert headers["content-md5"] == :crypto.hash(:md5, @sample_file_contents) |> Base.encode64() diff --git a/test/integration/container_integration_test.exs b/test/integration/container_integration_test.exs index ecb6c47..6505d35 100644 --- a/test/integration/container_integration_test.exs +++ b/test/integration/container_integration_test.exs @@ -2,7 +2,7 @@ defmodule Azurex.ContainerIntegrationTests do use ExUnit.Case, async: false alias Azurex.Blob.Container - @moduletag integration: true + @moduletag integration: true, azure_integration: true @integration_testing_container "integrationtestingcontainer" setup do From 14200ce8e311557ab3652f693075a38bcf900996 Mon Sep 17 00:00:00 2001 From: Henrik Rasmussen Date: Wed, 5 Mar 2025 20:21:54 +0100 Subject: [PATCH 4/5] Cleaned up and tested auth_method --- lib/azurex/blob/config.ex | 32 ++++++---------------- lib/azurex/blob/shared_access_signature.ex | 9 +++++- test/azurex/blob/config_test.exs | 22 +++++++++++---- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/lib/azurex/blob/config.ex b/lib/azurex/blob/config.ex index 62a7bc9..065a9eb 100644 --- a/lib/azurex/blob/config.ex +++ b/lib/azurex/blob/config.ex @@ -3,11 +3,6 @@ defmodule Azurex.Blob.Config do Azurex Blob Config """ - @missing_envs_error_msg """ - Azurex.Blob.Config: `storage_account_name` and `storage_account_key` - or `storage_account_connection_string` required. - """ - defp conf, do: Application.get_env(:azurex, __MODULE__, []) @doc """ @@ -41,7 +36,7 @@ defmodule Azurex.Blob.Config do case Keyword.get(conf(), :storage_account_name) do nil -> get_connection_string_value("AccountName") storage_account_name -> storage_account_name - end || raise @missing_envs_error_msg + end || raise "Azurex.Blob.Config: Missing storage account name" end defp try_account_key_env(nil) do @@ -76,12 +71,15 @@ defmodule Azurex.Blob.Config do nil _ -> - raise "Missing values for service principal #{Enum.join(missing_values, ", ")}" + raise "Azurex.Blob.Config: Missing values for service principal #{Enum.join(missing_values, ", ")}" end end defp try_service_principal(value), do: value + @doc """ + Investigate which authentication method is set. + """ @spec auth_method() :: {:service_principal, binary(), binary(), binary()} | {:account_key, binary()} def auth_method do @@ -90,26 +88,12 @@ defmodule Azurex.Blob.Config do |> try_account_key_conn_string |> try_service_principal || raise """ - Missing credentials settings. - Either by storage account key: `storage_account_name` and `storage_account_key` or `storage_account_connection_string` - Or by service principal: `storage_client_id`, `storage_client_secret` and `storage_tenant_id` + Azurex.Blob.Config: Missing credentials settings. + Either set storage account key with: `storage_account_key` or `storage_account_connection_string` + Or set service principal with: `storage_client_id`, `storage_client_secret` and `storage_tenant_id` """ end - @doc """ - Azure storage account access key. Base64 encoded, as provided by azure UI. - Required if `storage_account_connection_string` not set. - """ - @spec storage_account_key :: binary - def storage_account_key do - case Keyword.get(conf(), :storage_account_key) do - nil -> get_connection_string_value("AccountKey") - key -> key - end - |> Kernel.||(raise @missing_envs_error_msg) - |> Base.decode64!() - end - @doc """ Azure storage account connection string. Required if `storage_account_name` or `storage_account_key` not set. diff --git a/lib/azurex/blob/shared_access_signature.ex b/lib/azurex/blob/shared_access_signature.ex index d815a57..0baa04b 100644 --- a/lib/azurex/blob/shared_access_signature.ex +++ b/lib/azurex/blob/shared_access_signature.ex @@ -5,6 +5,7 @@ defmodule Azurex.Blob.SharedAccessSignature do Based on: https://learn.microsoft.com/en-us/rest/api/storageservices/create-service-sas """ + alias Azurex.Blob.Config @doc """ Generates a SAS url on a resource in a given container. @@ -32,6 +33,12 @@ defmodule Azurex.Blob.SharedAccessSignature do expiry = Keyword.get(opts, :expiry, {:second, 3600}) resource = Path.join(container, resource) + account_key = + case Config.auth_method() do + {:account_key, key} -> key + _ -> raise "Only account key authentication is supported for SAS" + end + token = build_token( resource_type, @@ -39,7 +46,7 @@ defmodule Azurex.Blob.SharedAccessSignature do {from, expiry}, permissions, Azurex.Blob.Config.storage_account_name(), - Azurex.Blob.Config.storage_account_key() + account_key ) "#{Path.join(base_url, resource)}?#{token}" diff --git a/test/azurex/blob/config_test.exs b/test/azurex/blob/config_test.exs index 8cac19b..6338ac2 100644 --- a/test/azurex/blob/config_test.exs +++ b/test/azurex/blob/config_test.exs @@ -35,21 +35,31 @@ defmodule Azurex.Blob.ConfigTest do end end - describe "storage_account_key/0" do - test "returns configured env" do + describe "auth_method/0" do + test "storage account key from storage_account_key" do put_config(storage_account_key: Base.encode64("sample key")) - assert storage_account_key() == "sample key" + assert auth_method() == {:account_key, "sample key"} end - test "returns based on storage_account_connection_string env" do + test "storage account key from storage_account_connection_string" do put_config(storage_account_connection_string: @sample_connection_string) - assert storage_account_key() == "cs_sample_key" + assert auth_method() == {:account_key, "cs_sample_key"} + end + + test "storage service principal" do + put_config( + storage_client_id: "test_client_id", + storage_client_secret: "test_secret", + storage_tenant_id: "test_tenant" + ) + + assert auth_method() == {:service_principal, "test_client_id", "test_secret", "test_tenant"} end test "error no env set" do put_config() - assert_raise RuntimeError, &storage_account_key/0 + assert_raise RuntimeError, &auth_method/0 end end From b2cec135eed332747095e825fdc102a184a53533 Mon Sep 17 00:00:00 2001 From: Henrik Rasmussen Date: Thu, 6 Mar 2025 20:16:32 +0100 Subject: [PATCH 5/5] Cache bearer token --- lib/azurex/authorization/service_principal.ex | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/azurex/authorization/service_principal.ex b/lib/azurex/authorization/service_principal.ex index 3eeee2a..194c8ac 100644 --- a/lib/azurex/authorization/service_principal.ex +++ b/lib/azurex/authorization/service_principal.ex @@ -1,12 +1,50 @@ defmodule Azurex.Authorization.ServicePrincipal do + @cache_key "azurex_bearer_token" + @cache_expiry_margin_seconds 10 def add_bearer_token(%HTTPoison.Request{} = request, client_id, client_secret, tenant_id) do - bearer_token = fetch_bearer_token(client_id, client_secret, tenant_id) + bearer_token = fetch_bearer_token_cached(client_id, client_secret, tenant_id) authorization = {"Authorization", "Bearer #{bearer_token}"} headers = [authorization | request.headers] struct(request, headers: headers) end + defp fetch_bearer_token_cached(client_id, client_secret, tenant_id) do + cache_key = @cache_key + + :ets.info(:bearer_token_cache) != :undefined || + :ets.new(:bearer_token_cache, [:named_table]) + + case :ets.lookup(:bearer_token_cache, cache_key) do + [{^cache_key, token, expiry}] -> + if expiry > System.os_time(:second) do + token + else + IO.inspect("Token expired, refreshing") + refresh_bearer_token_cache(client_id, client_secret, tenant_id) + end + + _ -> + refresh_bearer_token_cache(client_id, client_secret, tenant_id) + end + end + + defp refresh_bearer_token_cache(client_id, client_secret, tenant_id) do + token = fetch_bearer_token(client_id, client_secret, tenant_id) + expiry = extract_expiry_time(token) - @cache_expiry_margin_seconds + :ets.insert(:bearer_token_cache, {@cache_key, token, expiry}) + token + end + + defp extract_expiry_time(token) do + token + |> String.split(".") + |> Enum.at(1) + |> Base.decode64!() + |> Jason.decode!() + |> Map.get("exp") + end + def fetch_bearer_token(client_id, client_secret, tenant_id) do body = "grant_type=client_credentials&client_id=#{client_id}&client_secret=#{client_secret}&scope=https://storage.azure.com/.default"