From ba61e61b9cb6a0af4b958bf3fcd0fdf0d38c89a6 Mon Sep 17 00:00:00 2001 From: colmsnowplow Date: Fri, 21 Jun 2024 15:38:32 +0100 Subject: [PATCH] Fix component tests and use existing files for targets --- .../targets/eventhub-full-example.hcl | 12 +-- .../targets/http-full-example.hcl | 14 +++- .../targets/kafka-full-example.hcl | 6 +- .../configs/target-eventhub-extended.hcl | 15 ---- .../config/configs/target-eventhub-simple.hcl | 8 -- .../config/configs/target-http-extended.hcl | 17 ---- .../config/configs/target-http-simple.hcl | 7 -- .../config/configs/target-kafka-extended.hcl | 26 ------ .../config/configs/target-kafka-simple.hcl | 8 -- assets/test/config/configs/target-kinesis.hcl | 9 -- assets/test/config/configs/target-pubsub.hcl | 8 -- assets/test/config/configs/target-sqs.hcl | 9 -- config/component_test.go | 84 +++++++++++-------- 13 files changed, 69 insertions(+), 154 deletions(-) delete mode 100644 assets/test/config/configs/target-eventhub-extended.hcl delete mode 100644 assets/test/config/configs/target-eventhub-simple.hcl delete mode 100644 assets/test/config/configs/target-http-extended.hcl delete mode 100644 assets/test/config/configs/target-http-simple.hcl delete mode 100644 assets/test/config/configs/target-kafka-extended.hcl delete mode 100644 assets/test/config/configs/target-kafka-simple.hcl delete mode 100644 assets/test/config/configs/target-kinesis.hcl delete mode 100644 assets/test/config/configs/target-pubsub.hcl delete mode 100644 assets/test/config/configs/target-sqs.hcl diff --git a/assets/docs/configuration/targets/eventhub-full-example.hcl b/assets/docs/configuration/targets/eventhub-full-example.hcl index 355e9a7e..31f17d3a 100644 --- a/assets/docs/configuration/targets/eventhub-full-example.hcl +++ b/assets/docs/configuration/targets/eventhub-full-example.hcl @@ -13,21 +13,21 @@ target { max_auto_retries = 2 # Default presumes paid tier byte limit is 1MB (default: 1048576) - message_byte_limit = 1048576 + message_byte_limit = 1000000 # Chunk byte limit (default: 1048576) - chunk_byte_limit = 1048576 + chunk_byte_limit = 1000000 # Chunk message limit (default: 500) - chunk_message_limit = 500 + chunk_message_limit = 501 # The time (seconds) before context timeout (default: 20) - context_timeout_in_seconds = 20 + context_timeout_in_seconds = 21 # Default batch size of 1MB is the limit for Eventhub's high tier (default: 1048576) - batch_byte_limit = 1048576 + batch_byte_limit = 1000000 # Sets the eventHub message partition key, which is used by the EventHub client's batching strategy - set_eh_partition_key = true + set_eh_partition_key = false } } diff --git a/assets/docs/configuration/targets/http-full-example.hcl b/assets/docs/configuration/targets/http-full-example.hcl index 46ed9a54..d48846a1 100644 --- a/assets/docs/configuration/targets/http-full-example.hcl +++ b/assets/docs/configuration/targets/http-full-example.hcl @@ -5,14 +5,20 @@ target { # URL endpoint url = "https://acme.com/x" + # Maximum number of events that can go into one batched request (default: 20) + request_max_messages = 100 + # Byte limit for requests (default: 1048576) - byte_limit = 1048576 + request_byte_limit = 1000000 + + # Byte limit for individual messages (default: 1048576) + message_byte_limit = 1000000 # Request timeout in seconds (default: 5) - request_timeout_in_seconds = 5 + request_timeout_in_seconds = 2 # Content type for POST request (default: "application/json") - content_type = "application/json" + content_type = "text/html" # Optional headers to add to the request. # It is provided as a JSON string of key-value pairs (default: ""). @@ -30,7 +36,7 @@ target { cert_file = "myLocalhost.crt" # The optional key file for client authentication - key_file = "MyLocalhost.key" + key_file = "myLocalhost.key" # The optional certificate authority file for TLS client authentication ca_file = "myRootCA.crt" diff --git a/assets/docs/configuration/targets/kafka-full-example.hcl b/assets/docs/configuration/targets/kafka-full-example.hcl index 9fe5f379..41df612d 100644 --- a/assets/docs/configuration/targets/kafka-full-example.hcl +++ b/assets/docs/configuration/targets/kafka-full-example.hcl @@ -12,10 +12,10 @@ target { target_version = "2.7.0" # Max retries (default: 10) - max_retries = 10 + max_retries = 11 # Kafka default byte limit is 1MB (default: 1048576) - byte_limit = 1048576 + byte_limit = 1000000 # Whether to compress data (default: false). # Reduces network usage and increases latency. @@ -42,7 +42,7 @@ target { cert_file = "myLocalhost.crt" # The optional key file for client authentication - key_file = "MyLocalhost.key" + key_file = "myLocalhost.key" # The optional certificate authority file for TLS client authentication ca_file = "myRootCA.crt" diff --git a/assets/test/config/configs/target-eventhub-extended.hcl b/assets/test/config/configs/target-eventhub-extended.hcl deleted file mode 100644 index d8b2bcfe..00000000 --- a/assets/test/config/configs/target-eventhub-extended.hcl +++ /dev/null @@ -1,15 +0,0 @@ -# eventhub target extended config - -target { - use "eventhub" { - namespace = "testNamespace" - name = "testName" - max_auto_retries = 2 - message_byte_limit = 1000000 - chunk_byte_limit = 1000000 - chunk_message_limit = 501 - context_timeout_in_seconds = 21 - batch_byte_limit = 1000000 - set_eh_partition_key = false - } -} diff --git a/assets/test/config/configs/target-eventhub-simple.hcl b/assets/test/config/configs/target-eventhub-simple.hcl deleted file mode 100644 index 205ac45a..00000000 --- a/assets/test/config/configs/target-eventhub-simple.hcl +++ /dev/null @@ -1,8 +0,0 @@ -# eventhub target config - -target { - use "eventhub" { - namespace = "testNamespace" - name = "testName" - } -} diff --git a/assets/test/config/configs/target-http-extended.hcl b/assets/test/config/configs/target-http-extended.hcl deleted file mode 100644 index d29f4d8f..00000000 --- a/assets/test/config/configs/target-http-extended.hcl +++ /dev/null @@ -1,17 +0,0 @@ -# http target extended configuration - -target { - use "http" { - url = "testUrl" - byte_limit = 1000000 - request_timeout_in_seconds = 2 - content_type = "test/test" - headers = "{\"Accept-Language\":\"en-US\"}" - basic_auth_username = "testUsername" - basic_auth_password = "testPass" - cert_file = "myLocalhost.crt" - key_file = "MyLocalhost.key" - ca_file = "myRootCA.crt" - skip_verify_tls = true - } -} diff --git a/assets/test/config/configs/target-http-simple.hcl b/assets/test/config/configs/target-http-simple.hcl deleted file mode 100644 index f7617dd9..00000000 --- a/assets/test/config/configs/target-http-simple.hcl +++ /dev/null @@ -1,7 +0,0 @@ -# http target required configuration - -target { - use "http" { - url = "testUrl" - } -} diff --git a/assets/test/config/configs/target-kafka-extended.hcl b/assets/test/config/configs/target-kafka-extended.hcl deleted file mode 100644 index b325fcd7..00000000 --- a/assets/test/config/configs/target-kafka-extended.hcl +++ /dev/null @@ -1,26 +0,0 @@ -# kafka target extended configuration - -target { - use "kafka" { - brokers = "testBrokers" - topic_name = "testTopic" - target_version = "1.2.3" - max_retries = 11 - byte_limit = 1000000 - compress = true - wait_for_all = true - idempotent = true - enable_sasl = true - sasl_username = "testUsername" - sasl_password = "testPass" - sasl_algorithm = "sha256" - cert_file = "myLocalhost.crt" - key_file = "MyLocalhost.key" - ca_file = "myRootCA.crt" - skip_verify_tls = true - force_sync_producer = true - flush_frequency = 2 - flush_messages = 2 - flush_bytes = 2 - } -} diff --git a/assets/test/config/configs/target-kafka-simple.hcl b/assets/test/config/configs/target-kafka-simple.hcl deleted file mode 100644 index 29fa4780..00000000 --- a/assets/test/config/configs/target-kafka-simple.hcl +++ /dev/null @@ -1,8 +0,0 @@ -# kafka target required configuration - -target { - use "kafka" { - brokers = "testBrokers" - topic_name = "testTopic" - } -} diff --git a/assets/test/config/configs/target-kinesis.hcl b/assets/test/config/configs/target-kinesis.hcl deleted file mode 100644 index 763ca44b..00000000 --- a/assets/test/config/configs/target-kinesis.hcl +++ /dev/null @@ -1,9 +0,0 @@ -# kinesis target configuration - -target { - use "kinesis" { - stream_name = "testStream" - region = "eu-test-1" - role_arn = "xxx-test-role-arn" - } -} diff --git a/assets/test/config/configs/target-pubsub.hcl b/assets/test/config/configs/target-pubsub.hcl deleted file mode 100644 index e71b2a30..00000000 --- a/assets/test/config/configs/target-pubsub.hcl +++ /dev/null @@ -1,8 +0,0 @@ -# pubsub target configuration - -target { - use "pubsub" { - project_id = "testId" - topic_name = "testTopic" - } -} diff --git a/assets/test/config/configs/target-sqs.hcl b/assets/test/config/configs/target-sqs.hcl deleted file mode 100644 index 81dceb89..00000000 --- a/assets/test/config/configs/target-sqs.hcl +++ /dev/null @@ -1,9 +0,0 @@ -# config for sqs target - -target { - use "sqs" { - queue_name = "testQueue" - region = "eu-test-1" - role_arn = "xxx-test-role-arn" - } -} diff --git a/config/component_test.go b/config/component_test.go index 39445037..94c48389 100644 --- a/config/component_test.go +++ b/config/component_test.go @@ -13,6 +13,7 @@ package config import ( "errors" + "fmt" "path/filepath" "reflect" "testing" @@ -32,16 +33,15 @@ func TestCreateTargetComponentHCL(t *testing.T) { Expected interface{} }{ { - File: "target-sqs.hcl", + File: "targets/sqs-minimal-example.hcl", Plug: testSQSTargetAdapter(testSQSTargetFunc), Expected: &target.SQSTargetConfig{ - QueueName: "testQueue", - Region: "eu-test-1", - RoleARN: "xxx-test-role-arn", + QueueName: "mySqsQueue", + Region: "us-west-1", }, }, { - File: "target-eventhub-simple.hcl", + File: "targets/eventhub-minimal-example.hcl", Plug: testEventHubTargetAdapter(testEventHubTargetFunc), Expected: &target.EventHubConfig{ EventHubNamespace: "testNamespace", @@ -56,7 +56,7 @@ func TestCreateTargetComponentHCL(t *testing.T) { }, }, { - File: "target-eventhub-extended.hcl", + File: "targets/eventhub-full-example.hcl", Plug: testEventHubTargetAdapter(testEventHubTargetFunc), Expected: &target.EventHubConfig{ EventHubNamespace: "testNamespace", @@ -71,11 +71,11 @@ func TestCreateTargetComponentHCL(t *testing.T) { }, }, { - File: "target-http-simple.hcl", + File: "targets/http-minimal-example.hcl", Plug: testHTTPTargetAdapter(testHTTPTargetFunc), Expected: &target.HTTPTargetConfig{ - HTTPURL: "testUrl", - RequestMaxMessages: 1, + HTTPURL: "https://acme.com/x", + RequestMaxMessages: 20, RequestByteLimit: 1048576, MessageByteLimit: 1048576, RequestTimeoutInSeconds: 5, @@ -90,30 +90,35 @@ func TestCreateTargetComponentHCL(t *testing.T) { }, }, { - File: "target-http-extended.hcl", + File: "targets/http-full-example.hcl", Plug: testHTTPTargetAdapter(testHTTPTargetFunc), Expected: &target.HTTPTargetConfig{ - HTTPURL: "testUrl", - RequestMaxMessages: 10, + HTTPURL: "https://acme.com/x", + RequestMaxMessages: 100, RequestByteLimit: 1000000, MessageByteLimit: 1000000, RequestTimeoutInSeconds: 2, - ContentType: "test/test", + ContentType: "text/html", Headers: "{\"Accept-Language\":\"en-US\"}", - BasicAuthUsername: "testUsername", - BasicAuthPassword: "testPass", + BasicAuthUsername: "myUsername", + BasicAuthPassword: "myAuthPassword", + OAuth2ClientID: "myClientID", + OAuth2ClientSecret: "myClientSecret", + OAuth2RefreshToken: "myRefreshToken", + OAuth2TokenURL: "https://my.auth.server/token", CertFile: "myLocalhost.crt", - KeyFile: "MyLocalhost.key", + KeyFile: "myLocalhost.key", CaFile: "myRootCA.crt", SkipVerifyTLS: true, + DynamicHeaders: true, }, }, { - File: "target-kafka-simple.hcl", + File: "targets/kafka-minimal-example.hcl", Plug: testKafkaTargetAdapter(testKafkaTargetFunc), Expected: &target.KafkaConfig{ - Brokers: "testBrokers", - TopicName: "testTopic", + Brokers: "my-kafka-connection-string", + TopicName: "snowplow-enriched-good", TargetVersion: "", MaxRetries: 10, ByteLimit: 1048576, @@ -135,23 +140,23 @@ func TestCreateTargetComponentHCL(t *testing.T) { }, }, { - File: "target-kafka-extended.hcl", + File: "targets/kafka-full-example.hcl", Plug: testKafkaTargetAdapter(testKafkaTargetFunc), Expected: &target.KafkaConfig{ - Brokers: "testBrokers", - TopicName: "testTopic", - TargetVersion: "1.2.3", + Brokers: "my-kafka-connection-string", + TopicName: "snowplow-enriched-good", + TargetVersion: "2.7.0", MaxRetries: 11, ByteLimit: 1000000, Compress: true, WaitForAll: true, Idempotent: true, EnableSASL: true, - SASLUsername: "testUsername", - SASLPassword: "testPass", + SASLUsername: "mySaslUsername", + SASLPassword: "mySASLPassword", SASLAlgorithm: "sha256", CertFile: "myLocalhost.crt", - KeyFile: "MyLocalhost.key", + KeyFile: "myLocalhost.key", CaFile: "myRootCA.crt", SkipVerifyTLS: true, ForceSync: true, @@ -161,21 +166,21 @@ func TestCreateTargetComponentHCL(t *testing.T) { }, }, { - File: "target-kinesis.hcl", + File: "targets/kinesis-minimal-example.hcl", Plug: testKinesisTargetAdapter(testKinesisTargetFunc), Expected: &target.KinesisTargetConfig{ - StreamName: "testStream", - Region: "eu-test-1", - RoleARN: "xxx-test-role-arn", + StreamName: "my-stream", + Region: "us-west-1", + RoleARN: "", RequestMaxMessages: 500, }, }, { - File: "target-pubsub.hcl", + File: "targets/pubsub-minimal-example.hcl", Plug: testPubSubTargetAdapter(testPubSubTargetFunc), Expected: &target.PubSubTargetConfig{ - ProjectID: "testId", - TopicName: "testTopic", + ProjectID: "acme-project", + TopicName: "some-acme-topic", }, }, } @@ -184,9 +189,16 @@ func TestCreateTargetComponentHCL(t *testing.T) { t.Run(tt.File, func(t *testing.T) { assert := assert.New(t) - filename := filepath.Join(assets.AssetsRootDir, "test", "config", "configs", tt.File) + filename := filepath.Join(assets.AssetsRootDir, "docs", "configuration", tt.File) // TODO: change this to use the docs ones and remove the duplication t.Setenv("SNOWBRIDGE_CONFIG_FILE", filename) + // Set env vars referenced in configs + t.Setenv("CLIENT_ID", "myClientID") + t.Setenv("CLIENT_SECRET", "myClientSecret") + t.Setenv("REFRESH_TOKEN", "myRefreshToken") + t.Setenv("SASL_PASSWORD", "mySASLPassword") + t.Setenv("MY_AUTH_PASSWORD", "myAuthPassword") + c, err := NewConfig() assert.NotNil(c) if err != nil { @@ -201,6 +213,10 @@ func TestCreateTargetComponentHCL(t *testing.T) { result, err := c.CreateComponent(tt.Plug, decoderOpts) assert.NotNil(result) assert.Nil(err) + if err != nil { + fmt.Println("#####################################") + fmt.Println(err.Error()) + } if !reflect.DeepEqual(result, tt.Expected) { t.Errorf("GOT:\n%s\nEXPECTED:\n%s",