From 1f1f533a9733a7bb5adb665afcb002de66878b15 Mon Sep 17 00:00:00 2001 From: Stephen Firrincieli Date: Mon, 3 Feb 2020 15:34:32 -0500 Subject: [PATCH 1/5] Always write enhanced metrics to logs and set DD_ENHANCED_METRICS=true by default --- datadog_lambda/metric.py | 100 ++++++++++++++++++++------------------- datadog_lambda/tags.py | 24 ++++++++++ tests/test_metric.py | 32 ++++++------- tests/test_wrapper.py | 41 +++++++++------- 4 files changed, 117 insertions(+), 80 deletions(-) diff --git a/datadog_lambda/metric.py b/datadog_lambda/metric.py index ac235198..148ab979 100644 --- a/datadog_lambda/metric.py +++ b/datadog_lambda/metric.py @@ -4,7 +4,6 @@ # Copyright 2019 Datadog, Inc. import os -import sys import json import time import base64 @@ -13,8 +12,7 @@ import boto3 from datadog import api from datadog.threadstats import ThreadStats -from datadog_lambda import __version__ -from datadog_lambda.tags import get_enhanced_metrics_tags +from datadog_lambda.tags import get_enhanced_metrics_tags, tag_dd_lambda_layer ENHANCED_METRICS_NAMESPACE_PREFIX = "aws.lambda.enhanced" @@ -25,25 +23,6 @@ lambda_stats.start() -def _format_dd_lambda_layer_tag(): - """ - Formats the dd_lambda_layer tag, e.g., 'dd_lambda_layer:datadog-python27_1' - """ - runtime = "python{}{}".format(sys.version_info[0], sys.version_info[1]) - return "dd_lambda_layer:datadog-{}_{}".format(runtime, __version__) - - -def _tag_dd_lambda_layer(tags): - """ - Used by lambda_metric to insert the dd_lambda_layer tag - """ - dd_lambda_layer_tag = _format_dd_lambda_layer_tag() - if tags: - return tags + [dd_lambda_layer_tag] - else: - return [dd_lambda_layer_tag] - - def lambda_metric(metric_name, value, timestamp=None, tags=None): """ Submit a data point to Datadog distribution metrics. @@ -57,54 +36,79 @@ def lambda_metric(metric_name, value, timestamp=None, tags=None): periodically and at the end of the function execution in a background thread. """ - tags = _tag_dd_lambda_layer(tags) + tags = tag_dd_lambda_layer(tags) if os.environ.get("DD_FLUSH_TO_LOG", "").lower() == "true": - logger.debug("Sending metric %s to Datadog via log forwarder", metric_name) - print( - json.dumps( - { - "m": metric_name, - "v": value, - "e": timestamp or int(time.time()), - "t": tags, - } - ) - ) + write_metric_point_to_stdout(metric_name, value, timestamp, tags) else: logger.debug("Sending metric %s to Datadog via lambda layer", metric_name) lambda_stats.distribution(metric_name, value, timestamp=timestamp, tags=tags) +def write_metric_point_to_stdout(metric_name, value, timestamp=None, tags=[]): + """Writes the specified metric point to standard output + """ + logger.debug( + "Sending metric %s value %s to Datadog via log forwarder", metric_name, value + ) + print( + json.dumps( + { + "m": metric_name, + "v": value, + "e": timestamp or int(time.time()), + "t": tags, + } + ) + ) + + def are_enhanced_metrics_enabled(): """Check env var to find if enhanced metrics should be submitted + + Returns: + boolean for whether enhanced metrics are enabled """ - return os.environ.get("DD_ENHANCED_METRICS", "false").lower() == "true" + # DD_ENHANCED_METRICS defaults to true + return os.environ.get("DD_ENHANCED_METRICS", "true").lower() == "true" -def submit_invocations_metric(lambda_context): - """Increment aws.lambda.enhanced.invocations by 1 +def submit_enhanced_metric(metric_name, lambda_context): + """Submits the enhanced metric with the given name + + Args: + metric_name (str): metric name w/o "aws.lambda.enhanced." prefix i.e. "invocations" or "errors" + lambda_context (dict): Lambda context dict passed to the function by AWS """ if not are_enhanced_metrics_enabled(): + logger.debug( + "Not submitting enhanced metric %s because enhanced metrics are disabled" + ) return - lambda_metric( - "{}.invocations".format(ENHANCED_METRICS_NAMESPACE_PREFIX), + # Enhanced metrics are always written to logs + write_metric_point_to_stdout( + "{}.{}".format(ENHANCED_METRICS_NAMESPACE_PREFIX, metric_name), 1, tags=get_enhanced_metrics_tags(lambda_context), ) -def submit_errors_metric(lambda_context): - """Increment aws.lambda.enhanced.errors by 1 +def submit_invocations_metric(lambda_context): + """Increment aws.lambda.enhanced.invocations by 1, applying runtime, layer, and cold_start tags + + Args: + lambda_context (dict): Lambda context dict passed to the function by AWS """ - if not are_enhanced_metrics_enabled(): - return + submit_enhanced_metric("invocations", lambda_context) - lambda_metric( - "{}.errors".format(ENHANCED_METRICS_NAMESPACE_PREFIX), - 1, - tags=get_enhanced_metrics_tags(lambda_context), - ) + +def submit_errors_metric(lambda_context): + """Increment aws.lambda.enhanced.errors by 1, applying runtime, layer, and cold_start tags + + Args: + lambda_context (dict): Lambda context dict passed to the function by AWS + """ + submit_enhanced_metric("errors", lambda_context) # Set API Key and Host in the module, so they only set once per container diff --git a/datadog_lambda/tags.py b/datadog_lambda/tags.py index fa9f8ec2..8fd59eae 100644 --- a/datadog_lambda/tags.py +++ b/datadog_lambda/tags.py @@ -1,8 +1,30 @@ +import sys + from platform import python_version_tuple +from datadog_lambda import __version__ from datadog_lambda.cold_start import get_cold_start_tag +def _format_dd_lambda_layer_tag(): + """ + Formats the dd_lambda_layer tag, e.g., 'dd_lambda_layer:datadog-python27_1' + """ + runtime = "python{}{}".format(sys.version_info[0], sys.version_info[1]) + return "dd_lambda_layer:datadog-{}_{}".format(runtime, __version__) + + +def tag_dd_lambda_layer(tags): + """ + Used by lambda_metric to insert the dd_lambda_layer tag + """ + dd_lambda_layer_tag = _format_dd_lambda_layer_tag() + if tags: + return tags + [dd_lambda_layer_tag] + else: + return [dd_lambda_layer_tag] + + def parse_lambda_tags_from_arn(arn): """Generate the list of lambda tags based on the data in the arn Args: @@ -42,4 +64,6 @@ def get_enhanced_metrics_tags(lambda_context): get_cold_start_tag(), "memorysize:{}".format(lambda_context.memory_limit_in_mb), get_runtime_tag(), + _format_dd_lambda_layer_tag(), ] + diff --git a/tests/test_metric.py b/tests/test_metric.py index 7e553ae8..e7993551 100644 --- a/tests/test_metric.py +++ b/tests/test_metric.py @@ -1,38 +1,38 @@ import os import unittest + try: from unittest.mock import patch, call except ImportError: from mock import patch, call -from datadog_lambda.metric import ( - lambda_metric, - _format_dd_lambda_layer_tag, -) +from datadog_lambda.metric import lambda_metric +from datadog_lambda.tags import _format_dd_lambda_layer_tag class TestLambdaMetric(unittest.TestCase): - def setUp(self): - patcher = patch('datadog_lambda.metric.lambda_stats') + patcher = patch("datadog_lambda.metric.lambda_stats") self.mock_metric_lambda_stats = patcher.start() self.addCleanup(patcher.stop) def test_lambda_metric_tagged_with_dd_lambda_layer(self): - lambda_metric('test', 1) - lambda_metric('test', 1, 123, []) - lambda_metric('test', 1, tags=['tag1:test']) + lambda_metric("test", 1) + lambda_metric("test", 1, 123, []) + lambda_metric("test", 1, tags=["tag1:test"]) expected_tag = _format_dd_lambda_layer_tag() - self.mock_metric_lambda_stats.distribution.assert_has_calls([ - call('test', 1, timestamp=None, tags=[expected_tag]), - call('test', 1, timestamp=123, tags=[expected_tag]), - call('test', 1, timestamp=None, tags=['tag1:test', expected_tag]), - ]) + self.mock_metric_lambda_stats.distribution.assert_has_calls( + [ + call("test", 1, timestamp=None, tags=[expected_tag]), + call("test", 1, timestamp=123, tags=[expected_tag]), + call("test", 1, timestamp=None, tags=["tag1:test", expected_tag]), + ] + ) def test_lambda_metric_flush_to_log(self): - os.environ["DD_FLUSH_TO_LOG"] = 'True' + os.environ["DD_FLUSH_TO_LOG"] = "True" - lambda_metric('test', 1) + lambda_metric("test", 1) self.mock_metric_lambda_stats.distribution.assert_not_called() del os.environ["DD_FLUSH_TO_LOG"] diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index a45ca409..12e87835 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -62,6 +62,18 @@ def setUp(self): self.mock_python_version_tuple.return_value = ("2", "7", "10") self.addCleanup(patcher.stop) + patcher = patch("datadog_lambda.metric.write_metric_point_to_stdout") + self.mock_write_metric_point_to_stdout = patcher.start() + self.addCleanup(patcher.stop) + + patcher = patch("datadog_lambda.tags._format_dd_lambda_layer_tag") + self.mock_format_dd_lambda_layer_tag = patcher.start() + # Mock the layer version so we don't have to update tests on every version bump + self.mock_format_dd_lambda_layer_tag.return_value = ( + "dd_lambda_layer:datadog-python27_0.1.0" + ) + self.addCleanup(patcher.stop) + def test_datadog_lambda_wrapper(self): @datadog_lambda_wrapper def lambda_handler(event, context): @@ -111,8 +123,6 @@ def lambda_handler(event, context): del os.environ["DD_LOGS_INJECTION"] def test_invocations_metric(self): - os.environ["DD_ENHANCED_METRICS"] = "True" - @datadog_lambda_wrapper def lambda_handler(event, context): lambda_metric("test.metric", 100) @@ -121,7 +131,7 @@ def lambda_handler(event, context): lambda_handler(lambda_event, get_mock_context()) - self.mock_lambda_metric.assert_has_calls( + self.mock_write_metric_point_to_stdout.assert_has_calls( [ call( "aws.lambda.enhanced.invocations", @@ -133,16 +143,13 @@ def lambda_handler(event, context): "cold_start:true", "memorysize:256", "runtime:python2.7", + "dd_lambda_layer:datadog-python27_0.1.0", ], ) ] ) - del os.environ["DD_ENHANCED_METRICS"] - def test_errors_metric(self): - os.environ["DD_ENHANCED_METRICS"] = "True" - @datadog_lambda_wrapper def lambda_handler(event, context): raise RuntimeError() @@ -152,7 +159,7 @@ def lambda_handler(event, context): with self.assertRaises(RuntimeError): lambda_handler(lambda_event, get_mock_context()) - self.mock_lambda_metric.assert_has_calls( + self.mock_write_metric_point_to_stdout.assert_has_calls( [ call( "aws.lambda.enhanced.invocations", @@ -164,6 +171,7 @@ def lambda_handler(event, context): "cold_start:true", "memorysize:256", "runtime:python2.7", + "dd_lambda_layer:datadog-python27_0.1.0", ], ), call( @@ -176,16 +184,13 @@ def lambda_handler(event, context): "cold_start:true", "memorysize:256", "runtime:python2.7", + "dd_lambda_layer:datadog-python27_0.1.0", ], ), ] ) - del os.environ["DD_ENHANCED_METRICS"] - def test_enhanced_metrics_cold_start_tag(self): - os.environ["DD_ENHANCED_METRICS"] = "True" - @datadog_lambda_wrapper def lambda_handler(event, context): lambda_metric("test.metric", 100) @@ -200,7 +205,7 @@ def lambda_handler(event, context): lambda_event, get_mock_context(aws_request_id="second-request-id") ) - self.mock_lambda_metric.assert_has_calls( + self.mock_write_metric_point_to_stdout.assert_has_calls( [ call( "aws.lambda.enhanced.invocations", @@ -212,6 +217,7 @@ def lambda_handler(event, context): "cold_start:true", "memorysize:256", "runtime:python2.7", + "dd_lambda_layer:datadog-python27_0.1.0", ], ), call( @@ -224,14 +230,15 @@ def lambda_handler(event, context): "cold_start:false", "memorysize:256", "runtime:python2.7", + "dd_lambda_layer:datadog-python27_0.1.0", ], ), ] ) - del os.environ["DD_ENHANCED_METRICS"] - def test_no_enhanced_metrics_without_env_var(self): + os.environ["DD_ENHANCED_METRICS"] = "false" + @datadog_lambda_wrapper def lambda_handler(event, context): raise RuntimeError() @@ -241,4 +248,6 @@ def lambda_handler(event, context): with self.assertRaises(RuntimeError): lambda_handler(lambda_event, get_mock_context()) - self.mock_lambda_metric.assert_not_called() + self.mock_write_metric_point_to_stdout.assert_not_called() + + del os.environ["DD_ENHANCED_METRICS"] From 468b5e7267eef9ef0e4caacd79c8868457a94a4c Mon Sep 17 00:00:00 2001 From: Stephen Firrincieli Date: Mon, 3 Feb 2020 16:03:28 -0500 Subject: [PATCH 2/5] Fix log line --- datadog_lambda/metric.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datadog_lambda/metric.py b/datadog_lambda/metric.py index 148ab979..001ac38b 100644 --- a/datadog_lambda/metric.py +++ b/datadog_lambda/metric.py @@ -81,7 +81,8 @@ def submit_enhanced_metric(metric_name, lambda_context): """ if not are_enhanced_metrics_enabled(): logger.debug( - "Not submitting enhanced metric %s because enhanced metrics are disabled" + "Not submitting enhanced metric %s because enhanced metrics are disabled", + metric_name, ) return From a5e8a8092ea0e74f4ae3a0c351b2d596a3fe7ce8 Mon Sep 17 00:00:00 2001 From: Stephen Firrincieli Date: Mon, 3 Feb 2020 16:08:50 -0500 Subject: [PATCH 3/5] Fix formatting for flake8 --- datadog_lambda/metric.py | 2 +- datadog_lambda/tags.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/datadog_lambda/metric.py b/datadog_lambda/metric.py index 001ac38b..0c95650c 100644 --- a/datadog_lambda/metric.py +++ b/datadog_lambda/metric.py @@ -76,7 +76,7 @@ def submit_enhanced_metric(metric_name, lambda_context): """Submits the enhanced metric with the given name Args: - metric_name (str): metric name w/o "aws.lambda.enhanced." prefix i.e. "invocations" or "errors" + metric_name (str): metric name w/o enhanced prefix i.e. "invocations" or "errors" lambda_context (dict): Lambda context dict passed to the function by AWS """ if not are_enhanced_metrics_enabled(): diff --git a/datadog_lambda/tags.py b/datadog_lambda/tags.py index 8fd59eae..944818df 100644 --- a/datadog_lambda/tags.py +++ b/datadog_lambda/tags.py @@ -66,4 +66,3 @@ def get_enhanced_metrics_tags(lambda_context): get_runtime_tag(), _format_dd_lambda_layer_tag(), ] - From 572bf22dff308f3ed1b3bd70e02db331c0f4fbf4 Mon Sep 17 00:00:00 2001 From: Stephen Firrincieli Date: Mon, 3 Feb 2020 16:30:15 -0500 Subject: [PATCH 4/5] Bump version to 0.12.0 and update changelog --- CHANGELOG.md | 6 ++++++ datadog_lambda/__init__.py | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87e9c22e..6a49a930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # CHANGELOG +# Version 12 / 2020-02-03 + +- Defaults the DD_ENHANCED_METRICS option to `true` +- If DD_ENHANCED_METRICS is enabled, always writes enhanced metrics to stdout +- Note: if you previously had DD_ENHANCED_METRICS=true and did not set DD_FLUSH_TO_LOG=true, the enhanced metrics will no longer be submitted to Datadog synchonously; the metrics will now be written to logs. If you already have a Datadog Forwarder Lambda configured, that will read the enhanced metrics logs and submit the metrics asynchronously. If you do not have a Datadog Forwarder set up, you'll need to create one to get enhanced metrics into your Datadog account. See [Datadog Forwarder Lambda setup instructions](https://github.com/DataDog/datadog-serverless-functions/tree/master/aws/logs_monitoring). + # Version 11 / 2019-12-06 - Add python 3.8 support diff --git a/datadog_lambda/__init__.py b/datadog_lambda/__init__.py index bcfbe2b1..b3b4c5a8 100644 --- a/datadog_lambda/__init__.py +++ b/datadog_lambda/__init__.py @@ -1,9 +1,10 @@ # The minor version corresponds to the Lambda layer version. # E.g.,, version 0.5.0 gets packaged into layer version 5. -__version__ = '0.11.0' +__version__ = "0.12.0" import os import logging + logger = logging.getLogger(__name__) -logger.setLevel(logging.getLevelName(os.environ.get('DD_LOG_LEVEL', 'INFO').upper())) +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) From d617df20915b4e304dec0b6b138700a7c3edfdbf Mon Sep 17 00:00:00 2001 From: Stephen Firrincieli Date: Tue, 4 Feb 2020 11:20:00 -0500 Subject: [PATCH 5/5] Bump major version to 1 and update CHANGELOG --- CHANGELOG.md | 3 ++- datadog_lambda/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a49a930..20ab568e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ - Defaults the DD_ENHANCED_METRICS option to `true` - If DD_ENHANCED_METRICS is enabled, always writes enhanced metrics to stdout -- Note: if you previously had DD_ENHANCED_METRICS=true and did not set DD_FLUSH_TO_LOG=true, the enhanced metrics will no longer be submitted to Datadog synchonously; the metrics will now be written to logs. If you already have a Datadog Forwarder Lambda configured, that will read the enhanced metrics logs and submit the metrics asynchronously. If you do not have a Datadog Forwarder set up, you'll need to create one to get enhanced metrics into your Datadog account. See [Datadog Forwarder Lambda setup instructions](https://github.com/DataDog/datadog-serverless-functions/tree/master/aws/logs_monitoring). +- Breaking change: if you previously set the env var DD_ENHANCED_METRICS=true and did not set DD_FLUSH_TO_LOG=true, the enhanced metrics will no longer be submitted to Datadog synchronously; the metrics will now be written to logs. If you already have a Datadog Forwarder Lambda configured, that will read the enhanced metrics logs and submit the metrics asynchronously. If you do not have a Datadog Forwarder set up, you'll need to create one to get enhanced metrics into your Datadog account. See [Datadog Forwarder Lambda setup instructions](https://github.com/DataDog/datadog-serverless-functions/tree/master/aws/logs_monitoring). +- Because of the breaking change above, we've bumped the major package version so that this release is version 1.12.0. We've set the minor version to 12 so that the minor version of our package stays in alignment with the version of the layer we're releasing. # Version 11 / 2019-12-06 diff --git a/datadog_lambda/__init__.py b/datadog_lambda/__init__.py index b3b4c5a8..ba799966 100644 --- a/datadog_lambda/__init__.py +++ b/datadog_lambda/__init__.py @@ -1,6 +1,6 @@ # The minor version corresponds to the Lambda layer version. # E.g.,, version 0.5.0 gets packaged into layer version 5. -__version__ = "0.12.0" +__version__ = "1.12.0" import os