Release/0.8.0 (#110)

* Update dependencies * Bump up APIFW version to v0.8.0 * Bump up Go version to 1.21.13 * Fix GraphQL configuration issue * Add DNS cache * Add basic test for DNS LB cache
wallarm · Aug 19, 2024 · 56867e9 · 56867e9
1 parent 1f769c3
commit 56867e9
Show file tree

Hide file tree

Showing 30 changed files with 961 additions and 332 deletions.
diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml
@@ -51,7 +51,7 @@ jobs:
     needs:
       - draft-release
     env:
-      X_GO_DISTRIBUTION: "https://go.dev/dl/go1.21.12.linux-amd64.tar.gz"
+      X_GO_DISTRIBUTION: "https://go.dev/dl/go1.21.13.linux-amd64.tar.gz"
     strategy:
       matrix:
         include:
@@ -160,7 +160,7 @@ jobs:
     needs:
       - draft-release
     env:
-      X_GO_VERSION: "1.21.12"
+      X_GO_VERSION: "1.21.13"
     strategy:
       matrix:
         include:
@@ -267,11 +267,11 @@ jobs:
         include:
           - arch: armv6
             distro: bullseye
-            go_distribution: https://go.dev/dl/go1.21.12.linux-armv6l.tar.gz
+            go_distribution: https://go.dev/dl/go1.21.13.linux-armv6l.tar.gz
             artifact: armv6-libc
           - arch: aarch64
             distro: bullseye
-            go_distribution: https://go.dev/dl/go1.21.12.linux-arm64.tar.gz
+            go_distribution: https://go.dev/dl/go1.21.13.linux-arm64.tar.gz
             artifact: arm64-libc
           - arch: armv6
             distro: alpine_latest

diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-VERSION := 0.7.4
+VERSION := 0.8.0
 
 .DEFAULT_GOAL := build
 
@@ -13,14 +13,16 @@ tidy:
 	go mod vendor
 
 test:
-	go test ./... -count=1 -race -cover
+	go test ./... -count=1 -race -cover -run '^Test[^W]'
+	go test ./cmd/api-firewall/tests/main_dns_test.go
 
 bench:
 	GOMAXPROCS=1 go test -v -bench=. -benchtime=1000x -count 5 -benchmem -run BenchmarkWSGraphQL ./cmd/api-firewall/tests
 	GOMAXPROCS=4 go test -v -bench=. -benchtime=1000x -count 5 -benchmem -run BenchmarkWSGraphQL ./cmd/api-firewall/tests
 
 genmocks:
-	mockgen -source ./internal/platform/proxy/chainpool.go -destination ./internal/platform/proxy/httppool_mock.go -package proxy
+	mockgen -source ./internal/platform/proxy/chainpool.go -destination ./internal/platform/proxy/chainpool_mock.go -package proxy
+	mockgen -source ./internal/platform/proxy/dnscache.go -destination ./internal/platform/proxy/dnscache_mock.go -package proxy
 	mockgen -source ./internal/platform/storage/storage.go -destination ./internal/platform/storage/storage_mock.go -package storage
 	mockgen -source ./internal/platform/storage/updater/updater.go -destination ./internal/platform/storage/updater/updater_mock.go -package updater
 	mockgen -source ./internal/platform/proxy/ws.go -destination ./internal/platform/proxy/ws_mock.go -package proxy

diff --git a/cmd/api-firewall/internal/handlers/proxy/health.go b/cmd/api-firewall/internal/handlers/proxy/health.go
@@ -21,14 +21,14 @@ func (h Health) Readiness(ctx *fasthttp.RequestCtx) error {
 	status := "ok"
 	statusCode := fasthttp.StatusOK
 
-	reverseProxy, err := h.Pool.Get()
+	reverseProxy, ip, err := h.Pool.Get()
 	if err != nil {
 		status = "not ready"
 		statusCode = fasthttp.StatusInternalServerError
 	}
 
 	if reverseProxy != nil {
-		if err := h.Pool.Put(reverseProxy); err != nil {
+		if err := h.Pool.Put(ip, reverseProxy); err != nil {
 			status = "not ready"
 			statusCode = fasthttp.StatusInternalServerError
 		}

diff --git a/cmd/api-firewall/main.go b/cmd/api-firewall/main.go
@@ -4,6 +4,7 @@ import (
 	"expvar" // Register the expvar handlers
 	"fmt"
 	"mime"
+	"net"
 	"net/url"
 	"os"
 	"os/signal"
@@ -467,7 +468,7 @@ func runGraphQLMode(logger *logrus.Logger) error {
 		WriteTimeout:        cfg.Server.WriteTimeout,
 		DialTimeout:         cfg.Server.DialTimeout,
 	}
-	pool, err := proxy.NewChanPool(host, &options)
+	pool, err := proxy.NewChanPool(host, &options, nil)
 	if err != nil {
 		return errors.Wrap(err, "proxy pool init")
 	}
@@ -626,6 +627,9 @@ func runGraphQLMode(logger *logrus.Logger) error {
 			return errors.Wrap(err, "could not stop server gracefully")
 		}
 		logger.Infof("%s: %v: Completed shutdown", logPrefix, sig)
+
+		// Close proxy pool
+		pool.Close()
 	}
 
 	return nil
@@ -773,6 +777,16 @@ func runProxyMode(logger *logrus.Logger) error {
 		initialCap = 1
 	}
 
+	var dnsCacheResolver proxy.DNSCache
+
+	// init DNS resolver
+	if cfg.DNS.Cache {
+		dnsCacheResolver, err = proxy.NewDNSResolver(cfg.DNS.FetchTimeout, cfg.DNS.LookupTimeout, &net.Resolver{PreferGo: true}, logger)
+		if err != nil {
+			return errors.Wrap(err, "DNS cache resolver init")
+		}
+	}
+
 	options := proxy.Options{
 		InitialPoolCapacity: initialCap,
 		ClientPoolCapacity:  cfg.Server.ClientPoolCapacity,
@@ -782,8 +796,9 @@ func runProxyMode(logger *logrus.Logger) error {
 		ReadTimeout:         cfg.Server.ReadTimeout,
 		WriteTimeout:        cfg.Server.WriteTimeout,
 		DialTimeout:         cfg.Server.DialTimeout,
+		DNSConfig:           cfg.DNS,
 	}
-	pool, err := proxy.NewChanPool(host, &options)
+	pool, err := proxy.NewChanPool(host, &options, dnsCacheResolver)
 	if err != nil {
 		return errors.Wrap(err, "proxy pool init")
 	}

diff --git a/cmd/api-firewall/tests/main_dns_test.go b/cmd/api-firewall/tests/main_dns_test.go
@@ -0,0 +1,93 @@
+package tests
+
+import (
+	"context"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/foxcpp/go-mockdns"
+	"github.com/sirupsen/logrus"
+	"github.com/wallarm/api-firewall/internal/config"
+	"github.com/wallarm/api-firewall/internal/platform/proxy"
+)
+
+func TestWithoutRCCDNSCacheBasic(t *testing.T) {
+
+	logger := logrus.New()
+	logger.SetLevel(logrus.ErrorLevel)
+
+	var cfg = config.ProxyMode{
+		RequestValidation:         "BLOCK",
+		ResponseValidation:        "BLOCK",
+		CustomBlockStatusCode:     403,
+		AddValidationStatusHeader: false,
+		ShadowAPI: config.ShadowAPI{
+			ExcludeList: []int{404, 401},
+		},
+		DNS: config.DNS{
+			Cache:         true,
+			FetchTimeout:  1000 * time.Millisecond,
+			LookupTimeout: 400 * time.Millisecond,
+		},
+	}
+
+	srv, _ := mockdns.NewServer(map[string]mockdns.Zone{
+		"example.org.": {
+			A: []string{"1.2.3.4", "5.6.7.8"},
+		},
+	}, false)
+	defer srv.Close()
+
+	srUpdatedOrder, _ := mockdns.NewServer(map[string]mockdns.Zone{
+		"example.org.": {
+			A: []string{"5.6.7.8", "1.2.3.4"},
+		},
+	}, false)
+	defer srUpdatedOrder.Close()
+
+	r := &net.Resolver{}
+	srv.PatchNet(r)
+
+	dnsCache, err := proxy.NewDNSResolver(cfg.DNS.FetchTimeout, cfg.DNS.LookupTimeout, r, logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dnsCache.Stop()
+
+	addr, err := dnsCache.Fetch(context.Background(), "example.org")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if addr[0].String() != "1.2.3.4" {
+		t.Errorf("Incorrect response from local DNS server. Expected: 1.2.3.4 and got %s",
+			addr[0].String())
+	}
+
+	srUpdatedOrder.PatchNet(r)
+
+	time.Sleep(600 * time.Millisecond)
+
+	addr, err = dnsCache.Fetch(context.Background(), "example.org")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if addr[0].String() != "1.2.3.4" {
+		t.Errorf("Incorrect response from local DNS server. Expected: 1.2.3.4 and got %s",
+			addr[0].String())
+	}
+
+	time.Sleep(800 * time.Millisecond)
+
+	addr, err = dnsCache.Fetch(context.Background(), "example.org")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if addr[0].String() != "5.6.7.8" {
+		t.Errorf("Incorrect response from local DNS server. Expected: 5.6.7.8 and got %s",
+			addr[0].String())
+	}
+}
diff --git a/cmd/api-firewall/tests/main_graphql_bench_test.go b/cmd/api-firewall/tests/main_graphql_bench_test.go
@@ -134,7 +134,7 @@ func BenchmarkGraphQL(b *testing.B) {
 		WriteTimeout:        5 * time.Second,
 		DialTimeout:         5 * time.Second,
 	}
-	pool, err := proxy.NewChanPool(host, &options)
+	pool, err := proxy.NewChanPool(host, &options, nil)
 	if err != nil {
 		b.Fatalf("proxy pool init: %v", err)
 	}

diff --git a/cmd/api-firewall/tests/main_graphql_test.go b/cmd/api-firewall/tests/main_graphql_test.go
@@ -18,6 +18,7 @@ import (
 	"github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/valyala/fasthttp"
+
 	graphqlHandler "github.com/wallarm/api-firewall/cmd/api-firewall/internal/handlers/graphql"
 	"github.com/wallarm/api-firewall/internal/config"
 	"github.com/wallarm/api-firewall/internal/platform/denylist"
@@ -211,9 +212,9 @@ func (s *ServiceGraphQLTests) testGQLSuccess(t *testing.T) {
 		Request: *req,
 	}
 
-	s.proxy.EXPECT().Get().Return(s.client, nil).Times(1)
+	s.proxy.EXPECT().Get().Return(s.client, resolvedIP, nil).Times(1)
 	s.client.EXPECT().Do(gomock.Any(), gomock.Any()).SetArg(1, *resp).Times(1)
-	s.proxy.EXPECT().Put(s.client).Return(nil).Times(1)
+	s.proxy.EXPECT().Put(resolvedIP, s.client).Return(nil).Times(1)
 
 	handler(&reqCtx)
 
@@ -384,9 +385,9 @@ func (s *ServiceGraphQLTests) testGQLGETSuccess(t *testing.T) {
 		Request: *req,
 	}
 
-	s.proxy.EXPECT().Get().Return(s.client, nil).Times(1)
+	s.proxy.EXPECT().Get().Return(s.client, resolvedIP, nil).Times(1)
 	s.client.EXPECT().Do(gomock.Any(), gomock.Any()).SetArg(1, *resp).Times(1)
-	s.proxy.EXPECT().Put(s.client).Return(nil).Times(1)
+	s.proxy.EXPECT().Put(resolvedIP, s.client).Return(nil).Times(1)
 
 	handler(&reqCtx)
 
@@ -1715,9 +1716,9 @@ func (s *ServiceGraphQLTests) testGQLDuplicateFields(t *testing.T) {
 		Request: *req,
 	}
 
-	s.proxy.EXPECT().Get().Return(s.client, nil).AnyTimes()
+	s.proxy.EXPECT().Get().Return(s.client, resolvedIP, nil).AnyTimes()
 	s.client.EXPECT().Do(gomock.Any(), gomock.Any()).SetArg(1, *resp).AnyTimes()
-	s.proxy.EXPECT().Put(s.client).Return(nil).AnyTimes()
+	s.proxy.EXPECT().Put(resolvedIP, s.client).Return(nil).AnyTimes()
 
 	handler(&reqCtx)
 

diff --git a/cmd/api-firewall/tests/main_json_test.go b/cmd/api-firewall/tests/main_json_test.go
@@ -172,9 +172,9 @@ func (s *ServiceTests) testBasicObjJSONFieldValidation(t *testing.T) {
 		Request: *req,
 	}
 
-	s.proxy.EXPECT().Get().Return(s.client, nil)
+	s.proxy.EXPECT().Get().Return(s.client, resolvedIP, nil)
 	s.client.EXPECT().Do(gomock.Any(), gomock.Any()).SetArg(1, *resp)
-	s.proxy.EXPECT().Put(s.client).Return(nil)
+	s.proxy.EXPECT().Put(resolvedIP, s.client).Return(nil)
 
 	handler(&reqCtx)
 
@@ -219,9 +219,9 @@ func (s *ServiceTests) testBasicArrJSONFieldValidation(t *testing.T) {
 		Request: *req,
 	}
 
-	s.proxy.EXPECT().Get().Return(s.client, nil)
+	s.proxy.EXPECT().Get().Return(s.client, resolvedIP, nil)
 	s.client.EXPECT().Do(gomock.Any(), gomock.Any()).SetArg(1, *resp)
-	s.proxy.EXPECT().Put(s.client).Return(nil)
+	s.proxy.EXPECT().Put(resolvedIP, s.client).Return(nil)
 
 	handler(&reqCtx)