Merge branch 'master' of https://github.com/microsoft/opentelemetry-a…

…zure-monitor-python into v0.6b.x

README.md

@@ -1,3 +1,5 @@
+> **This repository has been moved to the [Azure SDK for Python](https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/monitor/azure-opentelemetry-exporter-azuremonitor) repository.** In order to improve discoverability and share common dependencies/tests, the OpenTelemetry Azure Monitor exporters for Python has moved to a common location containing all Azure SDKs. Please submit all issues and inquiries in that repository.
+
 # OpenTelemetry Azure Monitor
 
 [![Gitter chat](https://img.shields.io/gitter/room/Microsoft/azure-monitor-python)](https://gitter.im/Microsoft/azure-monitor-python)
@@ -47,11 +49,11 @@ with tracer.start_as_current_span('hello'):
     print('Hello World!')
 ```
 
-#### Integrations
+#### Instrumentations
 
-OpenTelemetry also supports several [integrations](https://github.com/open-telemetry/opentelemetry-python/tree/master/ext) which allows to integrate with third party libraries.
+OpenTelemetry also supports several [instrumentations](https://github.com/open-telemetry/opentelemetry-python/tree/master/instrumentation) which allows to instrument with third party libraries.
 
-This example shows how to integrate with the [requests](https://2.python-requests.org/en/master/)_ library.
+This example shows how to instrument with the [requests](https://2.python-requests.org/en/master/)_ library.
 
 * Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource).
 * Install the `requests` integration package using ``pip install opentelemetry-ext-http-requests``.

azure_monitor/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+## 0.6b.0
+Released 2021-01-28
+
+- Remove request metrics from auto-collection
+  ([#124](https://github.com/microsoft/opentelemetry-azure-monitor-python/pull/124))
+- Implement standard metrics for http dependency telemetry
+  ([#125](https://github.com/microsoft/opentelemetry-azure-monitor-python/pull/125))
+
 ## 0.5b.0
 Released 2020-09-24
 

azure_monitor/README.md

@@ -1,21 +1 @@
-# OpenTelemetry Azure Monitor SDKs and Exporters (Private preview)
-
-[![PyPI version](https://badge.fury.io/py/opentelemetry-azure-monitor.svg)](https://badge.fury.io/py/opentelemetry-azure-monitor)
-
-The OpenTelemetry Azure Monitor SDK and exporter are in private preview. They are not recommended for a production environment.
-
-## Installation
-
-```sh
-pip install opentelemetry-azure-monitor
-```
-
-# References
-
-[Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/)
-
-[OpenTelemetry Project](https://opentelemetry.io/)
-
-[OpenTelemetry Python Client](https://github.com/open-telemetry/opentelemetry-python)
-
-[Azure Monitor Python Gitter](https://gitter.im/Microsoft/azure-monitor-python)
+Deprecated, go for official version https://pypi.org/project/azure-opentelemetry-exporter-azuremonitor/.

azure_monitor/examples/metrics/auto_collector.py

@@ -19,7 +19,7 @@
 
 testing_label_set = {"environment": "testing"}
 
-# Automatically collect standard metrics
+# Automatically collect performance counters
 auto_collection = AutoCollection(meter=meter, labels=testing_label_set)
 
 metrics.get_meter_provider().start_pipeline(meter, exporter, 2)

azure_monitor/examples/metrics/standard.py

@@ -0,0 +1,33 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# pylint: disable=import-error
+# pylint: disable=no-member
+# pylint: disable=no-name-in-module
+import time
+
+import requests
+from opentelemetry import metrics
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk.metrics import MeterProvider
+
+from azure_monitor import AzureMonitorMetricsExporter
+
+# Use the default sdk implementation
+metrics.set_meter_provider(MeterProvider(stateful=False))
+
+# Track telemetry from the requests library
+RequestsInstrumentor().instrument()
+meter = RequestsInstrumentor().meter
+exporter = AzureMonitorMetricsExporter(
+    connection_string="InstrumentationKey=<INSTRUMENTATION KEY HERE>"
+)
+# Export standard metrics from requests library to Azure Monitor
+metrics.get_meter_provider().start_pipeline(meter, exporter, 5)
+
+for x in range(10):
+    for y in range(10):
+        requests.get("http://example.com")
+        time.sleep(2)
+    time.sleep(5)
+
+input("Press any key to exit...")

azure_monitor/src/azure_monitor/export/metrics/__init__.py

@@ -38,6 +38,10 @@ class AzureMonitorMetricsExporter(BaseExporter, MetricsExporter):
         options: :doc:`export.options` to allow configuration for the exporter
     """
 
+    def __init__(self, **options):
+        super().__init__(**options)
+        self.add_telemetry_processor(standard_metrics_processor)
+
     def export(
         self, metric_records: Sequence[MetricRecord]
     ) -> MetricsExportResult:
@@ -73,13 +77,19 @@ def _metric_to_envelope(
         )
         envelope.name = "Microsoft.ApplicationInsights.Metric"
         value = 0
+        _min = None
+        _max = None
+        count = None
         metric = metric_record.instrument
         if isinstance(metric, ValueObserver):
             # mmscl
             value = metric_record.aggregator.checkpoint.last
         elif isinstance(metric, ValueRecorder):
             # mmsc
-            value = metric_record.aggregator.checkpoint.count
+            value = metric_record.aggregator.checkpoint.sum
+            _min = metric_record.aggregator.checkpoint.min
+            _max = metric_record.aggregator.checkpoint.max
+            count = metric_record.aggregator.checkpoint.count
         else:
             # sum or lv
             value = metric_record.aggregator.checkpoint
@@ -90,6 +100,9 @@ def _metric_to_envelope(
             ns=metric.description,
             name=metric.name,
             value=value,
+            min=_min,
+            max=_max,
+            count=count,
             kind=protocol.DataPointType.MEASUREMENT.value,
         )
 
@@ -99,3 +112,43 @@ def _metric_to_envelope(
         data = protocol.MetricData(metrics=[data_point], properties=properties)
         envelope.data = protocol.Data(base_data=data, base_type="MetricData")
         return envelope
+
+
+def standard_metrics_processor(envelope):
+    data = envelope.data.base_data
+    if data.metrics:
+        properties = {}
+        point = data.metrics[0]
+        if point.name == "http.client.duration":
+            point.name = "Dependency duration"
+            point.kind = protocol.DataPointType.AGGREGATION.value
+            properties["_MS.MetricId"] = "dependencies/duration"
+            properties["_MS.IsAutocollected"] = "True"
+            properties["cloud/roleInstance"] = utils.azure_monitor_context.get(
+                "ai.cloud.roleInstance"
+            )
+            properties["cloud/roleName"] = utils.azure_monitor_context.get(
+                "ai.cloud.role"
+            )
+            properties["Dependency.Success"] = "False"
+            if data.properties.get("http.status_code"):
+                try:
+                    code = int(data.properties.get("http.status_code"))
+                    if 200 <= code < 400:
+                        properties["Dependency.Success"] = "True"
+                except ValueError:
+                    pass
+            # TODO: Check other properties if url doesn't exist
+            properties["dependency/target"] = data.properties.get("http.url")
+            properties["Dependency.Type"] = "HTTP"
+            properties["dependency/resultCode"] = data.properties.get(
+                "http.status_code"
+            )
+            # Won't need this once Azure Monitor supports histograms
+            # We can't actually get the individual buckets because the bucket
+            # collection must happen on the SDK side
+            properties["dependency/performanceBucket"] = ""
+            # TODO: OT does not have this in semantic conventions for trace
+            properties["operation/synthetic"] = ""
+        # TODO: Add other std. metrics as implemented
+        data.properties = properties

azure_monitor/src/azure_monitor/export/trace/__init__.py

@@ -27,6 +27,10 @@ class AzureMonitorSpanExporter(BaseExporter, SpanExporter):
         options: :doc:`export.options` to allow configuration for the exporter
     """
 
+    def __init__(self, **options):
+        super().__init__(**options)
+        self.add_telemetry_processor(indicate_processed_by_metric_extractors)
+
     def export(self, spans: Sequence[Span]) -> SpanExportResult:
         envelopes = list(map(self._span_to_envelope, spans))
         envelopes = list(
@@ -122,6 +126,7 @@ def convert_span_to_envelope(span: Span) -> protocol.Envelope:
                 "component" in span.attributes
                 and span.attributes["component"] == "http"
             ):
+                # TODO: check other component types (e.g. db)
                 data.type = "HTTP"
             if "http.url" in span.attributes:
                 url = span.attributes["http.url"]
@@ -157,3 +162,12 @@ def convert_span_to_envelope(span: Span) -> protocol.Envelope:
         data.properties["_MS.links"] = json.dumps(links)
     # TODO: tracestate, tags
     return envelope
+
+
+def indicate_processed_by_metric_extractors(envelope):
+    name = "Requests"
+    if envelope.data.base_type == "RemoteDependencyData":
+        name = "Dependencies"
+    envelope.data.base_data.properties["_MS.ProcessedByMetricExtractors"] = (
+        "(Name:'" + name + "',Ver:'1.1')"
+    )

azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py

@@ -24,8 +24,7 @@
 
 
 class AutoCollection:
-    """Starts auto collection of standard metrics, including performance,
-    dependency and request metrics.
+    """Starts auto collection of performance counters
 
     Args:
         meter: OpenTelemetry Meter
@@ -35,4 +34,3 @@ class AutoCollection:
     def __init__(self, meter: Meter, labels: Dict[str, str]):
         col_type = AutoCollectionType.PERF_COUNTER
         self._performance_metrics = PerformanceMetrics(meter, labels, col_type)
-        self._request_metrics = RequestMetrics(meter, labels, col_type)

azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py

@@ -91,7 +91,7 @@ def __init__(self, meter: Meter, labels: Dict[str, str]):
         )
 
     def _track_dependency_rate(self, observer: Observer) -> None:
-        """ Track Dependency rate
+        """Track Dependency rate
 
         Calculated by obtaining the number of outgoing requests made
         using the requests library within an elapsed time and dividing
@@ -121,7 +121,7 @@ def _track_dependency_rate(self, observer: Observer) -> None:
             observer.observe(last_result, self._labels)
 
     def _track_dependency_duration(self, observer: Observer) -> None:
-        """ Track Dependency average duration
+        """Track Dependency average duration
 
         Calculated by getting the time it takes to make an outgoing request
         and dividing over the amount of outgoing requests over an elapsed time.
@@ -149,7 +149,7 @@ def _track_dependency_duration(self, observer: Observer) -> None:
             observer.observe(last_average_duration, self._labels)
 
     def _track_failure_rate(self, observer: Observer) -> None:
-        """ Track Failed Dependency rate
+        """Track Failed Dependency rate
 
         Calculated by obtaining the number of failed outgoing requests made
         using the requests library within an elapsed time and dividing

azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py

@@ -42,7 +42,7 @@ def __init__(
         col_type = AutoCollectionType.LIVE_METRICS
         self._performance_metrics = PerformanceMetrics(meter, labels, col_type)
         self._dependency_metrics = DependencyMetrics(meter, labels)
-        self._request_metrics = RequestMetrics(meter, labels, col_type)
+        self._request_metrics = RequestMetrics(meter, labels)
         self._manager = LiveMetricsManager(
             meter, instrumentation_key, span_processor
         )

azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py

@@ -79,7 +79,7 @@ def __init__(
             )
 
     def _track_cpu(self, observer: Observer) -> None:
-        """ Track CPU time
+        """Track CPU time
 
         Processor time is defined as a float representing the current system
         wide CPU utilization minus idle CPU time as a percentage. Idle CPU
@@ -90,15 +90,15 @@ def _track_cpu(self, observer: Observer) -> None:
         observer.observe(100.0 - cpu_times_percent.idle, self._labels)
 
     def _track_memory(self, observer: Observer) -> None:
-        """ Track Memory
+        """Track Memory
 
         Available memory is defined as memory that can be given instantly to
         processes without the system going into swap.
         """
         observer.observe(psutil.virtual_memory().available, self._labels)
 
     def _track_process_cpu(self, observer: Observer) -> None:
-        """ Track Process CPU time
+        """Track Process CPU time
 
         Returns a derived gauge for the CPU usage for the current process.
         Return values range from 0.0 to 100.0 inclusive.
@@ -113,7 +113,7 @@ def _track_process_cpu(self, observer: Observer) -> None:
             logger.warning("Error handling get process cpu usage.")
 
     def _track_process_memory(self, observer: Observer) -> None:
-        """ Track Memory
+        """Track Memory
 
          Available memory is defined as memory that can be given instantly to
         processes without the system going into swap.
@@ -124,7 +124,7 @@ def _track_process_memory(self, observer: Observer) -> None:
             logger.warning("Error handling get process private bytes.")
 
     def _track_commited_memory(self, observer: Observer) -> None:
-        """ Track Commited Memory
+        """Track Commited Memory
 
         Available commited memory is defined as total memory minus available memory.
         """

azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py

@@ -9,8 +9,6 @@
 from opentelemetry.metrics import Meter, Observer
 from opentelemetry.sdk.metrics import UpDownSumObserver
 
-from azure_monitor.sdk.auto_collection.utils import AutoCollectionType
-
 _requests_lock = threading.Lock()
 logger = logging.getLogger(__name__)
 requests_map = dict()
@@ -79,29 +77,24 @@ class RequestMetrics:
     Args:
         meter: OpenTelemetry Meter
         labels: Dictionary of labels
-        collection_type: Standard or Live Metrics
     """
 
     def __init__(
-        self,
-        meter: Meter,
-        labels: Dict[str, str],
-        collection_type: AutoCollectionType,
+        self, meter: Meter, labels: Dict[str, str],
     ):
         self._meter = meter
         self._labels = labels
         # Patch the HTTPServer handler to track request information
         HTTPServer.__init__ = server_patch
 
-        if collection_type == AutoCollectionType.LIVE_METRICS:
-            meter.register_observer(
-                callback=self._track_request_failed_rate,
-                name="\\ApplicationInsights\\Requests Failed/Sec",
-                description="Incoming Requests Failed Rate",
-                unit="rps",
-                value_type=float,
-                observer_type=UpDownSumObserver,
-            )
+        meter.register_observer(
+            callback=self._track_request_failed_rate,
+            name="\\ApplicationInsights\\Requests Failed/Sec",
+            description="Incoming Requests Failed Rate",
+            unit="rps",
+            value_type=float,
+            observer_type=UpDownSumObserver,
+        )
         meter.register_observer(
             callback=self._track_request_duration,
             name="\\ApplicationInsights\\Request Duration",
@@ -120,7 +113,7 @@ def __init__(
         )
 
     def _track_request_duration(self, observer: Observer) -> None:
-        """ Track Request execution time
+        """Track Request execution time
 
         Calculated by getting the time it takes to make an incoming request
         and dividing over the amount of incoming requests over an elapsed time.
@@ -144,7 +137,7 @@ def _track_request_duration(self, observer: Observer) -> None:
             observer.observe(last_average_duration, self._labels)
 
     def _track_request_rate(self, observer: Observer) -> None:
-        """ Track Request execution rate
+        """Track Request execution rate
 
         Calculated by obtaining by getting the number of incoming requests
         made to an HTTPServer within an elapsed time and dividing that value
@@ -174,7 +167,7 @@ def _track_request_rate(self, observer: Observer) -> None:
             observer.observe(last_rate, self._labels)
 
     def _track_request_failed_rate(self, observer: Observer) -> None:
-        """ Track Request failed execution rate
+        """Track Request failed execution rate
 
         Calculated by obtaining by getting the number of failed incoming requests
         made to an HTTPServer within an elapsed time and dividing that value