Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added log analytics insights metrics format support #49

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Added log analytics insights metrics format support #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dfc8259
added log analytics insights metrics format support
himanshu219 Jan 6, 2023
90a5861
added support for parsed tags
himanshu219 Feb 8, 2023
2295a9d
added condition for ingesting tags
himanshu219 Feb 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion EventHubs/src/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ module.exports = function (context, eventHubMessages) {
var options ={ 'urlString':process.env.APPSETTING_SumoLogsEndpoint,'metadata':{}, 'MaxAttempts':3, 'RetryInterval':3000,'compress_data':true};


var transformer = new dataTransformer.Transformer();
var transformer = new dataTransformer.Transformer(context);
var messageArray = transformer.azureAudit(eventHubMessages);
if (messageArray.length !== 0) {
context.log("Sending: " + messageArray.length);
Expand Down
3 changes: 1 addition & 2 deletions EventHubs/src/index_metrics.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ module.exports = function (context, eventHubMessages) {
var options ={ 'urlString':process.env.APPSETTING_SumoLabsMetricEndpoint,'metadata':{}, 'MaxAttempts':3, 'RetryInterval':3000,'compress_data':true, 'metric_type':'carbon20'};


var transformer = new dataTransformer.Transformer();
var transformer = new dataTransformer.Transformer(context);
var messageArray = transformer.azureAudit(eventHubMessages);
var azureMetricArray = [];
var logRawArray = [];
Expand All @@ -58,7 +58,6 @@ module.exports = function (context, eventHubMessages) {
sumoMetricClient.addData(metricObjectArray);
context.log("Sending: " + metricObjectArray.length);


// handlers for success and failures
function failureHandler(msgArray,ctx) {
ctx.log("Failed to send metrics to Sumo");
Expand Down
93 changes: 67 additions & 26 deletions sumo-function-utils/lib/datatransformer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
* Class containing default data transformer
* @constructor
*/
function Transformer() {
function Transformer(context) {
this.context = context;
}

Transformer.prototype.azureAudit = function (data) {
Expand Down Expand Up @@ -40,31 +41,70 @@ Transformer.prototype.azureAudit = function (data) {
*/
Transformer.prototype.generateRawMetricObjectsFromAzureRawMetricRecord = function(msg, selectStatsForMetricFn) {
let finalMetricArray = [];
let stat_method;
// build core component
let core = Object.assign({},msg);
delete core.count;
delete core.total;
delete core.average;
delete core.maximum;
delete core.minimum;
delete core.metricName;
delete core.time;
core['timestamp'] = Math.ceil((new Date(msg.time)).getTime()/1000); //need to convert to epoch seconds
core['metric'] = msg['metricName'];

for (stat_method of selectStatsForMetricFn(msg)) {
if (msg && stat_method in msg) {
// in case some metrics don't have this statistic
let newDatapoint = Object.assign({},core);
newDatapoint['statistic']=stat_method;
newDatapoint['value']=msg[stat_method];
finalMetricArray.push(newDatapoint);
var self = this;
if (msg.SourceSystem && msg.SourceSystem === "Insights") {
let newDatapoint = Object.assign({}, msg);
// need to convert to epoch seconds
newDatapoint['timestamp'] = Math.ceil((new Date(msg.TimeGenerated)).getTime()/1000);
newDatapoint['metric'] = msg.Name;
newDatapoint['namespace'] = msg.Namespace;
newDatapoint['value'] = msg.Val;
if (process.env.APPSETTING_EnableTagsInLogAnalytics && process.env.APPSETTING_EnableTagsInLogAnalytics === "true") {
try {
tagObj = JSON.parse(msg.Tags);
newDatapoint = Object.assign(newDatapoint, tagObj);
// self.context.log(tagObj);
// self.context.log(newDatapoint);
} catch (e) {
self.context.log("Error in TagsParsing", e);
}
}
delete newDatapoint.Tags
delete newDatapoint.Val;
delete newDatapoint.Namespace;
delete newDatapoint.Name;
delete newDatapoint.TimeGenerated;
finalMetricArray.push(newDatapoint);
} else {
let stat_method;
// build core component
let core = Object.assign({},msg);
delete core.count;
delete core.total;
delete core.average;
delete core.maximum;
delete core.minimum;
delete core.metricName;
delete core.time;
// need to convert to epoch seconds
core['timestamp'] = Math.ceil((new Date(msg.time)).getTime()/1000);
core['metric'] = msg['metricName'];
for (stat_method of selectStatsForMetricFn(msg)) {
if (msg && stat_method in msg) {
// in case some metrics don't have this statistic
let newDatapoint = Object.assign({},core);
newDatapoint['statistic']=stat_method;
newDatapoint['value']=msg[stat_method];
finalMetricArray.push(newDatapoint);
}
}
}


return finalMetricArray
}

Transformer.prototype.getMetricKeyValue = function(key, metricObject) {
let kv = ""
if (metricObject && metricObject[key] && (typeof metricObject[key] === "string") && (metricObject[key].indexOf(" ") > -1)) {
// replacing string values with spaces with quotes
kv = key + '=' + metricObject[key].replace(/\s/g, "_");
} else {
kv = key + '=' + metricObject[key];
}
return kv;
}


/**
* Function to prepare a metric object to be ready to be digested by a SumoMetricClient.
Expand Down Expand Up @@ -92,9 +132,9 @@ Transformer.prototype.prepareMetricObject = function(metricObject,format) {
if (Object.keys(metricObject).length > 0) {
const msg_keys = Object.keys(metricObject);
// now construct the Dimensions and put in _sumo_meta_data
let dimension_string = msg_keys[0] + '=' + metricObject[msg_keys[0]];
let dimension_string = this.getMetricKeyValue(msg_keys[0], metricObject);
for (var i = 1; i < msg_keys.length; i++) {
dimension_string += ',' + msg_keys[i] + '=' + metricObject[msg_keys[i]];
dimension_string += ',' + this.getMetricKeyValue(msg_keys[i], metricObject);
}
if (meta_data['X-Sumo-Dimensions'] && meta_data['X-Sumo-Dimensions'] !== '') {
meta_data['X-Sumo-Dimensions'] += ',' + dimension_string;
Expand All @@ -119,8 +159,9 @@ Transformer.prototype.prepareMetricObject = function(metricObject,format) {
delete metricObject['_sumo_meta_data'];
// now convert all other key pair
for (const key of Object.keys(metricObject)) {
metric_string += key + '=' + metricObject[key] + ' ';
metric_string += this.getMetricKeyValue(key, metricObject) + ' ';
}
// this.context.log("MetricString: ", metric_string);
// now put back other metric name, value and timestamp the meta_data
metric_string += 'metric=' + metric_name + ' ' + metric_value + ' ' + metric_timestamp;
// put back meta_data if it's there
Expand All @@ -145,9 +186,9 @@ Transformer.prototype.prepareMetricObject = function(metricObject,format) {
Transformer.prototype.generateMetricObjectsFromAzureRawData = function(azureMetricArray,selectStatsForMetricFn,format) {
let finalMetricArray = [];
for (const msg of azureMetricArray) {
let metricArray = Transformer.prototype.generateRawMetricObjectsFromAzureRawMetricRecord(msg,selectStatsForMetricFn);
let metricArray = this.generateRawMetricObjectsFromAzureRawMetricRecord(msg,selectStatsForMetricFn);
for (let metricObj of metricArray) {
finalMetricArray.push(Transformer.prototype.prepareMetricObject(metricObj,format));
finalMetricArray.push(this.prepareMetricObject(metricObj,format));
}
}
return finalMetricArray;
Expand Down
7 changes: 5 additions & 2 deletions sumo-function-utils/lib/sumometricclient.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,9 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {

if (curOptions.compress_data) {
curOptions.headers['Content-Encoding'] = 'gzip';

zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
var payload = msgArray.join('\n');
// self.context.log("Sending Uncompressed Payload size: " + Buffer.byteLength(payload, 'utf-8'));
zlib.gzip(payload,function(e,compressed_data){
if (!e) {
sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
.then(()=> {
Expand All @@ -181,12 +182,14 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
self.messagesFailed += msgArray.length;
self.messagesAttempted += msgArray.length;
self.context.log("Failed to send after retries: " + self.MaxAttempts + " " + JSON.stringify(err) + ' messagesAttempted: ' + self.messagesAttempted + ' messagesReceived: ' + self.messagesReceived);
// self.context.log(payload);
self.failure_callback(msgArray,self.context);
});
} else {
self.messagesFailed += msgArray.length;
self.messagesAttempted += msgArray.length;
self.context.log("Failed to gzip: " + JSON.stringify(e) + ' messagesAttempted: ' + self.messagesAttempted + ' messagesReceived: ' + self.messagesReceived);
// self.context.log(payload);
self.failure_callback(msgArray,self.context);
}
});
Expand Down