Add support for customized deny responses to the RLQS filter. (#38299)

Add support for customized response bodies, headers and
status codes when the rate_limit_quota filter is denying a query. This
was already in the RateLimitQuotaService proto but not implemented in
the filter itself.

Risk Level: Minor
Testing: Unit, integration
Docs Changes:
Release Notes:
Platform Specific Features:

Signed-off-by: Brian Surber <[email protected]>

source/extensions/filters/http/rate_limit_quota/filter.cc

@@ -11,6 +11,7 @@
 #include "envoy/http/header_map.h"
 #include "envoy/matcher/matcher.h"
 #include "envoy/stream_info/stream_info.h"
+#include "envoy/type/v3/http_status.pb.h"
 #include "envoy/type/v3/ratelimit_strategy.pb.h"
 
 #include "source/common/common/logger.h"
@@ -31,9 +32,10 @@ namespace RateLimitQuota {
 
 const char kBucketMetadataNamespace[] = "envoy.extensions.http_filters.rate_limit_quota.bucket";
 
+using envoy::extensions::filters::http::rate_limit_quota::v3::RateLimitQuotaBucketSettings;
 using envoy::type::v3::RateLimitStrategy;
-using NoAssignmentBehavior = envoy::extensions::filters::http::rate_limit_quota::v3::
-    RateLimitQuotaBucketSettings::NoAssignmentBehavior;
+using NoAssignmentBehavior = RateLimitQuotaBucketSettings::NoAssignmentBehavior;
+using DenyResponseSettings = RateLimitQuotaBucketSettings::DenyResponseSettings;
 
 // Returns whether or not to allow a request based on the no-assignment-behavior
 // & populates an action.
@@ -44,10 +46,31 @@ bool noAssignmentBehaviorShouldAllow(const NoAssignmentBehavior& no_assignment_b
                RateLimitStrategy::DENY_ALL);
 }
 
+// Translate from the HttpStatus Code enum to the Envoy::Http::Code enum.
+inline Envoy::Http::Code getDenyResponseCode(const DenyResponseSettings& settings) {
+  if (!settings.has_http_status()) {
+    return Envoy::Http::Code::TooManyRequests;
+  }
+  return static_cast<Envoy::Http::Code>(static_cast<uint64_t>(settings.http_status().code()));
+}
+
+inline std::function<void(Http::ResponseHeaderMap&)>
+addDenyResponseHeadersCb(const DenyResponseSettings& settings) {
+  if (settings.response_headers_to_add().empty())
+    return nullptr;
+  // Headers copied from settings for thread-safety.
+  return [headers_to_add = settings.response_headers_to_add()](Http::ResponseHeaderMap& headers) {
+    for (const envoy::config::core::v3::HeaderValueOption& header : headers_to_add) {
+      headers.addCopy(Http::LowerCaseString(header.header().key()), header.header().value());
+    }
+  };
+}
+
 Http::FilterHeadersStatus sendDenyResponse(Http::StreamDecoderFilterCallbacks* cb,
-                                           Envoy::Http::Code code,
+                                           const DenyResponseSettings& settings,
                                            StreamInfo::CoreResponseFlag flag) {
-  cb->sendLocalReply(code, "", nullptr, absl::nullopt, "");
+  cb->sendLocalReply(getDenyResponseCode(settings), settings.http_body().value(),
+                     addDenyResponseHeadersCb(settings), absl::nullopt, "");
   cb->streamInfo().setResponseFlag(flag);
   return Envoy::Http::FilterHeadersStatus::StopIteration;
 }
@@ -94,10 +117,14 @@ Http::FilterHeadersStatus RateLimitQuotaFilter::decodeHeaders(Http::RequestHeade
   }
   callbacks_->streamInfo().setDynamicMetadata(kBucketMetadataNamespace, bucket_log);
 
+  // Settings needed if a cached bucket or default behavior decides to deny.
+  const DenyResponseSettings& deny_response_settings =
+      match_action.bucketSettings().deny_response_settings();
+
   std::shared_ptr<CachedBucket> cached_bucket = client_->getBucket(bucket_id);
   if (cached_bucket != nullptr) {
     // Found the cached bucket entry.
-    return processCachedBucket(*cached_bucket);
+    return processCachedBucket(deny_response_settings, *cached_bucket);
   }
 
   // New buckets should have a configured default action pulled from
@@ -151,7 +178,7 @@ Http::FilterHeadersStatus RateLimitQuotaFilter::decodeHeaders(Http::RequestHeade
     return Envoy::Http::FilterHeadersStatus::Continue;
   }
 
-  return sendDenyResponse(callbacks_, Envoy::Http::Code::TooManyRequests,
+  return sendDenyResponse(callbacks_, deny_response_settings,
                           StreamInfo::CoreResponseFlag::ResponseFromCacheFilter);
 }
 
@@ -237,7 +264,9 @@ bool RateLimitQuotaFilter::shouldAllowRequest(const CachedBucket& cached_bucket)
   return true; // Unreachable.
 }
 
-Http::FilterHeadersStatus RateLimitQuotaFilter::processCachedBucket(CachedBucket& cached_bucket) {
+Http::FilterHeadersStatus
+RateLimitQuotaFilter::processCachedBucket(const DenyResponseSettings& deny_response_settings,
+                                          CachedBucket& cached_bucket) {
   // The QuotaUsage of a cached bucket should never be null. If it is due to a
   // bug, this will crash.
   std::shared_ptr<QuotaUsage> quota_usage = cached_bucket.quota_usage;
@@ -250,7 +279,7 @@ Http::FilterHeadersStatus RateLimitQuotaFilter::processCachedBucket(CachedBucket
   incrementAtomic(quota_usage->num_requests_denied);
   // TODO(tyxia) Build the customized response based on
   // `DenyResponseSettings` if it is configured.
-  return sendDenyResponse(callbacks_, Envoy::Http::Code::TooManyRequests,
+  return sendDenyResponse(callbacks_, deny_response_settings,
                           StreamInfo::CoreResponseFlag::ResponseFromCacheFilter);
 }
 

source/extensions/filters/http/rate_limit_quota/filter.h

@@ -30,6 +30,8 @@ using QuotaAssignmentAction = ::envoy::service::rate_limit_quota::v3::RateLimitQ
 using FilterConfig =
     envoy::extensions::filters::http::rate_limit_quota::v3::RateLimitQuotaFilterConfig;
 using FilterConfigConstSharedPtr = std::shared_ptr<const FilterConfig>;
+using DenyResponseSettings = envoy::extensions::filters::http::rate_limit_quota::v3::
+    RateLimitQuotaBucketSettings::DenyResponseSettings;
 
 /**
  * Possible async results for a limit call.
@@ -71,7 +73,8 @@ class RateLimitQuotaFilter : public Http::PassThroughFilter,
   }
 
 private:
-  Http::FilterHeadersStatus processCachedBucket(CachedBucket& cached_bucket);
+  Http::FilterHeadersStatus processCachedBucket(const DenyResponseSettings& deny_response_settings,
+                                                CachedBucket& cached_bucket);
   bool shouldAllowRequest(const CachedBucket& cached_bucket);
 
   FilterConfigConstSharedPtr config_;

test/extensions/filters/http/rate_limit_quota/BUILD

@@ -121,6 +121,7 @@ envoy_extension_cc_test(
         "//test/test_common:status_utility_lib",
         "//test/test_common:test_runtime_lib",
         "//test/test_common:utility_lib",
+        "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/http/rate_limit_quota/v3:pkg_cc_proto",
         "@envoy_api//envoy/service/rate_limit_quota/v3:pkg_cc_proto",
         "@envoy_api//envoy/type/v3:pkg_cc_proto",

test/extensions/filters/http/rate_limit_quota/filter_test.cc

@@ -428,6 +428,35 @@ BucketId bucketIdFromMap(const absl::flat_hash_map<std::string, std::string>& bu
   return bucket_id;
 }
 
+TEST_F(FilterTest, DecodeHeaderWithValidOnNoMatchDenyWithSettings) {
+  addMatcherConfig(MatcherConfigType::ValidOnNoMatchConfig);
+  createFilter();
+  constructMismatchedRequestHeader();
+
+  absl::flat_hash_map<std::string, std::string> expected_bucket_ids({
+      {"on_no_match_name", "on_no_match_value"},
+      {"on_no_match_name_2", "on_no_match_value_2"},
+  });
+  BucketId bucket_id = bucketIdFromMap(expected_bucket_ids);
+  size_t bucket_id_hash = MessageUtil::hash(bucket_id);
+  BucketAction no_assignment_action;
+  no_assignment_action.mutable_quota_assignment_action()
+      ->mutable_rate_limit_strategy()
+      ->set_blanket_rule(RateLimitStrategy::DENY_ALL);
+  *no_assignment_action.mutable_bucket_id() = bucket_id;
+
+  // Default behavior is set to deny with custom settings in the bucket
+  // matcher's `no_assignment_behavior` & `deny_response_settings`.
+  EXPECT_CALL(*mock_local_client_, getBucket(bucket_id_hash)).WillOnce(Return(nullptr));
+  EXPECT_CALL(*mock_local_client_, createBucket(ProtoEqIgnoreRepeatedFieldOrdering(bucket_id),
+                                                bucket_id_hash, ProtoEq(no_assignment_action), _,
+                                                std::chrono::milliseconds::zero(), false))
+      .WillOnce(Return());
+
+  Http::FilterHeadersStatus status = filter_->decodeHeaders(default_headers_, false);
+  EXPECT_EQ(status, Envoy::Http::FilterHeadersStatus::StopIteration);
+}
+
 TEST_F(FilterTest, DecodeHeadersWithoutCachedAssignment) {
   addMatcherConfig(MatcherConfigType::Valid);
   createFilter();

test/extensions/filters/http/rate_limit_quota/integration_test.cc

@@ -6,6 +6,7 @@
 #include <utility>
 #include <vector>
 
+#include "envoy/config/core/v3/base.pb.h"
 #include "envoy/event/dispatcher.h"
 #include "envoy/extensions/filters/http/rate_limit_quota/v3/rate_limit_quota.pb.h"
 #include "envoy/http/codec.h"
@@ -62,13 +63,16 @@ MATCHER_P2(ProtoEqIgnoringFieldAndOrdering, expected,
 
 using BlanketRule = envoy::type::v3::RateLimitStrategy::BlanketRule;
 using envoy::type::v3::RateLimitStrategy;
+using DenyResponseSettings = envoy::extensions::filters::http::rate_limit_quota::v3::
+    RateLimitQuotaBucketSettings::DenyResponseSettings;
 
 struct ConfigOption {
   bool valid_rlqs_server = true;
   absl::optional<BlanketRule> no_assignment_blanket_rule = std::nullopt;
   bool unsupported_no_assignment_strategy = false;
   absl::optional<RateLimitStrategy> fallback_rate_limit_strategy = std::nullopt;
   int fallback_ttl_sec = 15;
+  absl::optional<DenyResponseSettings> deny_response_settings = std::nullopt;
 };
 
 // These tests exercise the rate limit quota filter through Envoy's integration test
@@ -165,6 +169,11 @@ class RateLimitQuotaIntegrationTest : public Event::TestUsingSimulatedTime,
             ->set_seconds(config_option.fallback_ttl_sec);
       }
 
+      if (config_option.deny_response_settings.has_value()) {
+        *mutable_bucket_settings.mutable_deny_response_settings() =
+            *config_option.deny_response_settings;
+      }
+
       mutable_config->PackFrom(mutable_bucket_settings);
       proto_config_.mutable_bucket_matchers()->MergeFrom(matcher);
 
@@ -205,11 +214,27 @@ class RateLimitQuotaIntegrationTest : public Event::TestUsingSimulatedTime,
 
   void TearDown() override { cleanUp(); }
 
-  bool expectDeniedRequest(int expected_status_code) {
+  bool expectDeniedRequest(int expected_status_code,
+                           std::vector<std::pair<std::string, std::string>> expected_headers = {},
+                           std::string expected_body = "") {
     if (!response_->waitForEndStream())
       return false;
     EXPECT_TRUE(response_->complete());
     EXPECT_EQ(response_->headers().getStatusValue(), absl::StrCat(expected_status_code));
+
+    // Check for expected headers & body if set.
+    for (const auto& [key, value] : expected_headers) {
+      Http::HeaderMap::GetResult result = response_->headers().get(Http::LowerCaseString(key));
+      if (result.empty()) {
+        EXPECT_FALSE(result.empty());
+        continue;
+      }
+      EXPECT_THAT(result[0]->value().getStringView(), testing::StrEq(value));
+    }
+    if (!expected_body.empty()) {
+      EXPECT_THAT(response_->body(), testing::StrEq(expected_body));
+    }
+
     cleanupUpstreamAndDownstream();
     return true;
   }
@@ -594,6 +619,117 @@ TEST_P(RateLimitQuotaIntegrationTest, MultiSameRequestNoAssignmentDenyAll) {
   EXPECT_EQ(usage.num_requests_denied(), 2);
 }
 
+TEST_P(RateLimitQuotaIntegrationTest, MultiSameRequestNoAssignmentDenyAllWithSettings) {
+  ConfigOption option;
+  option.no_assignment_blanket_rule = RateLimitStrategy::DENY_ALL;
+  option.deny_response_settings = DenyResponseSettings();
+  option.deny_response_settings->mutable_http_status()->set_code(
+      envoy::type::v3::StatusCode::Forbidden);
+  *option.deny_response_settings->mutable_http_body()->mutable_value() =
+      "Denied by no-assignment behavior.";
+  envoy::config::core::v3::HeaderValueOption* new_header =
+      option.deny_response_settings->mutable_response_headers_to_add()->Add();
+  new_header->mutable_header()->set_key("custom-denial-header-key");
+  new_header->mutable_header()->set_value("custom-denial-header-value");
+
+  initializeConfig(option);
+  HttpIntegrationTest::initialize();
+  absl::flat_hash_map<std::string, std::string> custom_headers = {{"environment", "staging"},
+                                                                  {"group", "envoy"}};
+
+  for (int i = 0; i < 3; ++i) {
+    sendClientRequest(&custom_headers);
+
+    if (i == 0) {
+      // Start the gRPC stream to RLQS server on the first request.
+      ASSERT_TRUE(grpc_upstreams_[0]->waitForHttpConnection(*dispatcher_, rlqs_connection_));
+      ASSERT_TRUE(rlqs_connection_->waitForNewStream(*dispatcher_, rlqs_stream_));
+      rlqs_stream_->startGrpcStream();
+
+      // Expect an initial report.
+      RateLimitQuotaUsageReports reports;
+      ASSERT_TRUE(rlqs_stream_->waitForGrpcMessage(*dispatcher_, reports));
+
+      // Verify the usage report content.
+      ASSERT_THAT(reports.bucket_quota_usages_size(), 1);
+      const auto& usage = reports.bucket_quota_usages(0);
+      // The request is denied by no_assignment_behavior.
+      EXPECT_EQ(usage.num_requests_allowed(), 0);
+      EXPECT_EQ(usage.num_requests_denied(), 1);
+    }
+
+    // No response sent so filter should fallback to NoAssignmentBehavior
+    // (deny-all here with a customized response code + body + headers).
+    // Verify the response to downstream.
+    EXPECT_TRUE(expectDeniedRequest(403,
+                                    {{"custom-denial-header-key", "custom-denial-header-value"}},
+                                    "Denied by no-assignment behavior."));
+  }
+
+  simTime().advanceTimeWait(std::chrono::seconds(report_interval_sec_));
+  RateLimitQuotaUsageReports reports;
+  ASSERT_TRUE(rlqs_stream_->waitForGrpcMessage(*dispatcher_, reports));
+  ASSERT_THAT(reports.bucket_quota_usages_size(), 1);
+  const auto& usage = reports.bucket_quota_usages(0);
+  // The request is denied by no_assignment_behavior.
+  EXPECT_EQ(usage.num_requests_allowed(), 0);
+  EXPECT_EQ(usage.num_requests_denied(), 2);
+}
+
+TEST_P(RateLimitQuotaIntegrationTest, MultiSameRequestNoAssignmentDenyAllWithEmptyBodySettings) {
+  ConfigOption option;
+  option.no_assignment_blanket_rule = RateLimitStrategy::DENY_ALL;
+  option.deny_response_settings = DenyResponseSettings();
+  option.deny_response_settings->mutable_http_status()->set_code(
+      envoy::type::v3::StatusCode::Forbidden);
+  envoy::config::core::v3::HeaderValueOption* new_header =
+      option.deny_response_settings->mutable_response_headers_to_add()->Add();
+  new_header->mutable_header()->set_key("custom-denial-header-key");
+  new_header->mutable_header()->set_value("custom-denial-header-value");
+
+  initializeConfig(option);
+  HttpIntegrationTest::initialize();
+  absl::flat_hash_map<std::string, std::string> custom_headers = {{"environment", "staging"},
+                                                                  {"group", "envoy"}};
+
+  for (int i = 0; i < 3; ++i) {
+    sendClientRequest(&custom_headers);
+
+    if (i == 0) {
+      // Start the gRPC stream to RLQS server on the first request.
+      ASSERT_TRUE(grpc_upstreams_[0]->waitForHttpConnection(*dispatcher_, rlqs_connection_));
+      ASSERT_TRUE(rlqs_connection_->waitForNewStream(*dispatcher_, rlqs_stream_));
+      rlqs_stream_->startGrpcStream();
+
+      // Expect an initial report.
+      RateLimitQuotaUsageReports reports;
+      ASSERT_TRUE(rlqs_stream_->waitForGrpcMessage(*dispatcher_, reports));
+
+      // Verify the usage report content.
+      ASSERT_THAT(reports.bucket_quota_usages_size(), 1);
+      const auto& usage = reports.bucket_quota_usages(0);
+      // The request is denied by no_assignment_behavior.
+      EXPECT_EQ(usage.num_requests_allowed(), 0);
+      EXPECT_EQ(usage.num_requests_denied(), 1);
+    }
+
+    // No response sent so filter should fallback to NoAssignmentBehavior
+    // (deny-all here with a customized response code + body + headers).
+    // Verify the response to downstream.
+    EXPECT_TRUE(
+        expectDeniedRequest(403, {{"custom-denial-header-key", "custom-denial-header-value"}}, ""));
+  }
+
+  simTime().advanceTimeWait(std::chrono::seconds(report_interval_sec_));
+  RateLimitQuotaUsageReports reports;
+  ASSERT_TRUE(rlqs_stream_->waitForGrpcMessage(*dispatcher_, reports));
+  ASSERT_THAT(reports.bucket_quota_usages_size(), 1);
+  const auto& usage = reports.bucket_quota_usages(0);
+  // The request is denied by no_assignment_behavior.
+  EXPECT_EQ(usage.num_requests_allowed(), 0);
+  EXPECT_EQ(usage.num_requests_denied(), 2);
+}
+
 TEST_P(RateLimitQuotaIntegrationTest, MultiDifferentRequestNoAssignementAllowAll) {
   ConfigOption option;
   option.no_assignment_blanket_rule = RateLimitStrategy::ALLOW_ALL;

test/extensions/filters/http/rate_limit_quota/test_utils.h

@@ -109,6 +109,17 @@ inline constexpr absl::string_view OnNoMatchConfig = R"EOF(
         deny_response_settings:
           grpc_status:
             code: 8
+          http_status:
+            code: 403
+          http_body:
+            value: "Test-rejection-body"
+          response_headers_to_add:
+            header:
+              key: "test-onnomatch-header"
+              value: "test-onnomatch-value"
+        no_assignment_behavior:
+          fallback_rate_limit:
+            blanket_rule: DENY_ALL
         expired_assignment_behavior:
           fallback_rate_limit:
             blanket_rule: ALLOW_ALL