Skip to content

Commit

Permalink
feat(daemon): Add Log Event harvest functionality (#495)
Browse files Browse the repository at this point in the history 
* feat(src,axiom): initial changes to support log event integration testing

* fix(src): remove LogEventData HarvestLimit metric

* feat(src): integrate log events into harvest functionality

* feat(src): add log event support to harvest and processor tests

* feat(daemon): implement log event harvest

* test(daemon): initial add of log event harvest tests

* fix(daemon): add log events to flatbuffers for Transaction.go
  • Loading branch information
@bduranleau-nr
bduranleau-nr authored Aug 3, 2022
1 parent 941fd49 commit 5546944
Show file tree
Hide file tree
Showing 15 changed files with 192 additions and 16 deletions.
1 change: 1 addition & 0 deletions axiom/tests/.gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ test_header
test_helgrind
test_json
test_labels
test_log_event
test_logging
test_logging_parallel
test_math
Expand Down
1 change: 1 addition & 0 deletions protocol/flatbuffers/protocol.fbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ table Transaction {
error_events: [Event]; // added in the 5.1 PHP agent release
sampling_priority: double; // added in the 8.2 PHP agent release
span_events: [Event];
log_events: [Event]; // added in the 10.1 PHP agent release
}

union MessageBody { App, AppReply, Transaction, SpanBatch }
Expand Down
16 changes: 11 additions & 5 deletions src/newrelic/app_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ func TestConnectPayloadEncoded(t *testing.T) {
// A valid span event max samples stored value configured from the agent should
// propagate through and be sent to the collector
info.AgentEventLimits.SpanEventConfig.Limit = 2323
info.AgentEventLimits.LogEventConfig.Limit = 4545

pid := 123
expected := `[` +
Expand All @@ -375,7 +376,7 @@ func TestConnectPayloadEncoded(t *testing.T) {
`"metadata":{"NEW_RELIC_METADATA_ONE":"one","NEW_RELIC_METADATA_TWO":"two"},` +
`"identifier":"one;two",` +
`"utilization":{"metadata_version":1,"logical_processors":22,"total_ram_mib":1000,"hostname":"some_host"},` +
`"event_harvest_config":{"report_period_ms":60000,"harvest_limits":{"error_event_data":100,"analytic_event_data":10000,"custom_event_data":10000,"span_event_data":2323}}` +
`"event_harvest_config":{"report_period_ms":60000,"harvest_limits":{"error_event_data":100,"analytic_event_data":10000,"custom_event_data":10000,"span_event_data":2323,"log_event_data":4545}}` +
`}` +
`]`

Expand All @@ -389,6 +390,7 @@ func TestConnectPayloadEncoded(t *testing.T) {
// An invalid span event max samples stored value configured from the agent should
// propagate defaults through and be sent to the collector
info.AgentEventLimits.SpanEventConfig.Limit = 12345
info.AgentEventLimits.LogEventConfig.Limit = 45678

pid = 123
expected = `[` +
Expand All @@ -406,7 +408,10 @@ func TestConnectPayloadEncoded(t *testing.T) {
`"metadata":{"NEW_RELIC_METADATA_ONE":"one","NEW_RELIC_METADATA_TWO":"two"},` +
`"identifier":"one;two",` +
`"utilization":{"metadata_version":1,"logical_processors":22,"total_ram_mib":1000,"hostname":"some_host"},` +
`"event_harvest_config":{"report_period_ms":60000,"harvest_limits":{"error_event_data":100,"analytic_event_data":10000,"custom_event_data":10000,"span_event_data":` + strconv.Itoa(limits.MaxSpanMaxEvents) + `}}` +
`"event_harvest_config":{"report_period_ms":60000,` +
`"harvest_limits":{"error_event_data":100,"analytic_event_data":10000,"custom_event_data":10000,` +
`"span_event_data":` + strconv.Itoa(limits.MaxSpanMaxEvents) +
`,"log_event_data":` + strconv.Itoa(limits.MaxLogMaxEvents) + `}}` +
`}` +
`]`

Expand All @@ -419,6 +424,7 @@ func TestConnectPayloadEncoded(t *testing.T) {

// an empty string for the HostDisplayName should not produce JSON
info.AgentEventLimits.SpanEventConfig.Limit = 1001
info.AgentEventLimits.LogEventConfig.Limit = 1002
info.HostDisplayName = ""
expected = `[` +
`{` +
Expand All @@ -434,7 +440,7 @@ func TestConnectPayloadEncoded(t *testing.T) {
`"metadata":{"NEW_RELIC_METADATA_ONE":"one","NEW_RELIC_METADATA_TWO":"two"},` +
`"identifier":"one;two",` +
`"utilization":{"metadata_version":1,"logical_processors":22,"total_ram_mib":1000,"hostname":"some_host"},` +
`"event_harvest_config":{"report_period_ms":60000,"harvest_limits":{"error_event_data":100,"analytic_event_data":10000,"custom_event_data":10000,"span_event_data":1001}}` +
`"event_harvest_config":{"report_period_ms":60000,"harvest_limits":{"error_event_data":100,"analytic_event_data":10000,"custom_event_data":10000,"span_event_data":1001,"log_event_data":1002}}` +
`}` +
`]`

Expand All @@ -461,7 +467,7 @@ func TestConnectPayloadEncoded(t *testing.T) {
`"metadata":{},` +
`"identifier":"one;two",` +
`"utilization":{"metadata_version":1,"logical_processors":22,"total_ram_mib":1000,"hostname":"some_host"},` +
`"event_harvest_config":{"report_period_ms":60000,"harvest_limits":{"error_event_data":100,"analytic_event_data":10000,"custom_event_data":10000,"span_event_data":1001}}` +
`"event_harvest_config":{"report_period_ms":60000,"harvest_limits":{"error_event_data":100,"analytic_event_data":10000,"custom_event_data":10000,"span_event_data":1001,"log_event_data":1002}}` +
`}` +
`]`

Expand All @@ -488,7 +494,7 @@ func TestConnectPayloadEncoded(t *testing.T) {
`"metadata":{},` +
`"identifier":"one;two",` +
`"utilization":{"metadata_version":1,"logical_processors":22,"total_ram_mib":1000,"hostname":"some_host"},` +
`"event_harvest_config":{"report_period_ms":60000,"harvest_limits":{"error_event_data":100,"analytic_event_data":10000,"custom_event_data":10000,"span_event_data":1001}}` +
`"event_harvest_config":{"report_period_ms":60000,"harvest_limits":{"error_event_data":100,"analytic_event_data":10000,"custom_event_data":10000,"span_event_data":1001,"log_event_data":1002}}` +
`}` +
`]`

Expand Down
1 change: 1 addition & 0 deletions src/newrelic/collector/collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ const (
CommandConnect = "connect"
CommandPreconnect = "preconnect"
CommandSpanEvents = "span_event_data"
CommandLogEvents = "log_event_data"
)

const (
Expand Down
26 changes: 26 additions & 0 deletions src/newrelic/collector/event_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ type rawHarvestLimits struct {
AnalyticEventData *int `json:"analytic_event_data"`
CustomEventData *int `json:"custom_event_data"`
SpanEventData *int `json:"span_event_data"`
LogEventData *int `json:"log_event_data"`
}

// rawEventHarvestConfig is the wire representation of the event_harvest_config,
Expand Down Expand Up @@ -58,6 +59,7 @@ type EventConfigs struct {
AnalyticEventConfig Event
CustomEventConfig Event
SpanEventConfig Event
LogEventConfig Event
}

// EventHarvestConfig represents the event_harvest_config object used by the daemon
Expand Down Expand Up @@ -103,6 +105,7 @@ func (daemonConfig EventHarvestConfig) MarshalJSON() ([]byte, error) {
AnalyticEventData: &daemonConfig.EventConfigs.AnalyticEventConfig.Limit,
CustomEventData: &daemonConfig.EventConfigs.CustomEventConfig.Limit,
SpanEventData: &daemonConfig.EventConfigs.SpanEventConfig.Limit,
LogEventData: &daemonConfig.EventConfigs.LogEventConfig.Limit,
}

rawConfig := rawEventHarvestConfig{
Expand Down Expand Up @@ -269,6 +272,18 @@ func (daemonConfig *EventHarvestConfig) UnmarshalJSON(b []byte) error {
return err
}

harvestConfig.LogEventConfig.Limit,
harvestConfig.LogEventConfig.ReportPeriod,
err = getEventConfig(
rawLimits.LogEventData,
daemonConfig.ReportPeriod,
limits.MaxLogMaxEvents,
limits.DefaultReportPeriod)
if err != nil {
log.Infof("Unexpected negative Log event limit %d", rawLimits.LogEventData)
return err
}

// Copy the harvest limits in.
daemonConfig.EventConfigs = harvestConfig

Expand All @@ -292,6 +307,14 @@ func NewHarvestLimits(agentLimits *EventConfigs) EventConfigs {
}
}

logEventLimit := limits.MaxLogMaxEvents
if agentLimits != nil {
if (agentLimits.LogEventConfig.Limit < limits.MaxLogMaxEvents) &&
(agentLimits.LogEventConfig.Limit >= 0) {
logEventLimit = agentLimits.LogEventConfig.Limit
}
}

return EventConfigs{
ErrorEventConfig: Event{
Limit: limits.MaxErrorEvents,
Expand All @@ -305,6 +328,9 @@ func NewHarvestLimits(agentLimits *EventConfigs) EventConfigs {
SpanEventConfig: Event{
Limit: spanEventLimit,
},
LogEventConfig: Event{
Limit: logEventLimit,
},
}
}

Expand Down
26 changes: 21 additions & 5 deletions src/newrelic/collector/event_data_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestDurationToMillisecondsSuccess(t *testing.T) {
}

func TestMarshalJSONEvent(t *testing.T) {
expected := `{"report_period_ms":20000,"harvest_limits":{"error_event_data":0,"analytic_event_data":0,"custom_event_data":0,"span_event_data":0}}`
expected := `{"report_period_ms":20000,"harvest_limits":{"error_event_data":0,"analytic_event_data":0,"custom_event_data":0,"span_event_data":0,"log_event_data":0}}`
output, err := json.Marshal(EventHarvestConfig{ReportPeriod: 20 * time.Second})
if err != nil {
t.Errorf("unexpected error when marshalling a valid config: %v", err)
Expand Down Expand Up @@ -113,6 +113,7 @@ func CreateEventConfig(reportPeriod time.Duration) EventConfigs {
eventConfig.AnalyticEventConfig.ReportPeriod = reportPeriod
eventConfig.CustomEventConfig.ReportPeriod = reportPeriod
eventConfig.SpanEventConfig.ReportPeriod = reportPeriod
eventConfig.LogEventConfig.ReportPeriod = reportPeriod

return eventConfig
}
Expand Down Expand Up @@ -190,12 +191,12 @@ func TestUnmarshalJSONEventSuccess(t *testing.T) {
},
{
name: "null harvest limits",
json: `{"harvest_limits":{"error_event_data":null,"analytic_event_data":null,"custom_event_data":null,"span_event_data":null}}`,
json: `{"harvest_limits":{"error_event_data":null,"analytic_event_data":null,"custom_event_data":null,"span_event_data":null,"log_event_data":null}}`,
expected: NewDefaultEventHarvestConfig(),
},
{
name: "zero harvest limits",
json: `{"harvest_limits":{"error_event_data":0,"analytic_event_data":0,"custom_event_data":0,"span_event_data":0}}`,
json: `{"harvest_limits":{"error_event_data":0,"analytic_event_data":0,"custom_event_data":0,"span_event_data":0,"log_event_data":0}}`,
expected: EventHarvestConfig{
ReportPeriod: defaultReportPeriodMs,
EventConfigs: EventConfigs{
Expand All @@ -215,12 +216,16 @@ func TestUnmarshalJSONEventSuccess(t *testing.T) {
Limit: 0,
ReportPeriod: defaultReportPeriodMs,
},
LogEventConfig: Event{
Limit: 0,
ReportPeriod: defaultReportPeriodMs,
},
},
},
},
{
name: "valid harvest limits",
json: `{"harvest_limits":{"error_event_data":21,"analytic_event_data":22,"custom_event_data":23,"span_event_data":24}}`,
json: `{"harvest_limits":{"error_event_data":21,"analytic_event_data":22,"custom_event_data":23,"span_event_data":24,"log_event_data":25}}`,
expected: EventHarvestConfig{
ReportPeriod: defaultReportPeriodMs,
EventConfigs: EventConfigs{
Expand All @@ -240,12 +245,16 @@ func TestUnmarshalJSONEventSuccess(t *testing.T) {
Limit: 24,
ReportPeriod: defaultReportPeriodMs,
},
LogEventConfig: Event{
Limit: 25,
ReportPeriod: defaultReportPeriodMs,
},
},
},
},
{
name: "valid harvest limits and report period",
json: `{"report_period_ms":42,"harvest_limits":{"error_event_data":21,"analytic_event_data":22,"custom_event_data":23,"span_event_data":24}}`,
json: `{"report_period_ms":42,"harvest_limits":{"error_event_data":21,"analytic_event_data":22,"custom_event_data":23,"span_event_data":24,"log_event_data": 25}}`,
expected: EventHarvestConfig{
ReportPeriod: 42 * time.Millisecond,
EventConfigs: EventConfigs{
Expand All @@ -265,6 +274,10 @@ func TestUnmarshalJSONEventSuccess(t *testing.T) {
Limit: 24,
ReportPeriod: 42 * time.Millisecond,
},
LogEventConfig: Event{
Limit: 25,
ReportPeriod: 42 * time.Millisecond,
},
},
},
},
Expand All @@ -291,6 +304,9 @@ func TestUnmarshalJSONEventSuccess(t *testing.T) {
SpanEventConfig: Event{
Limit: 4,
},
LogEventConfig: Event{
Limit: 5,
},
},
}
testUnmarshalJSONEventCaseSuccess(t, input.name, "junk context", input.json, &ehc, &input.expected)
Expand Down
10 changes: 10 additions & 0 deletions src/newrelic/commands.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,16 @@ func (t FlatTxn) AggregateInto(h *Harvest) {
}
}

if n := txn.LogEventsLength(); n > 0 {
var e protocol.Event

for i := 0; i < n; i++ {
txn.LogEvents(&e, i)
data := copySlice(e.Data())
h.LogEvents.AddEventFromData(data, samplingPriority)
}
}

if trace := txn.Trace(nil); trace != nil {
data := trace.Data()
tt := &TxnTrace{
Expand Down
11 changes: 10 additions & 1 deletion src/newrelic/harvest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ func NewHarvest(now time.Time, hl collector.EventConfigs) *Harvest {
CustomEvents: NewCustomEvents(hl.CustomEventConfig.Limit),
ErrorEvents: NewErrorEvents(hl.ErrorEventConfig.Limit),
SpanEvents: NewSpanEvents(hl.SpanEventConfig.Limit),
LogEvents: NewLogEvents(hl.LogEventConfig.Limit),
commandsProcessed: 0,
pidSet: make(map[int]struct{}),
httpErrorSet: make(map[int]float64),
Expand All @@ -58,7 +59,8 @@ func (h *Harvest) empty() bool {
h.Metrics.Empty() &&
h.SlowSQLs.Empty() &&
h.TxnEvents.Empty() &&
h.TxnTraces.Empty()
h.TxnTraces.Empty() &&
h.LogEvents.Empty()
}

func createTraceObserverMetrics(to *infinite_tracing.TraceObserver, metrics *MetricTable) {
Expand Down Expand Up @@ -157,6 +159,11 @@ func (h *Harvest) createFinalMetrics(harvestLimits collector.EventHarvestConfig,
h.Metrics.AddCount("Supportability/SpanEvent/TotalEventsSent", "", h.SpanEvents.analyticsEvents.NumSaved(), Forced)
h.createEndpointAttemptsMetric(h.SpanEvents.Cmd(), h.SpanEvents.analyticsEvents.NumFailedAttempts())

// Log Events Supportability Metrics
h.Metrics.AddCount("Supportability/LogEvent/TotalEventsSeen", "", h.LogEvents.analyticsEvents.NumSeen(), Forced)
h.Metrics.AddCount("Supportability/LogEvent/TotalEventsSent", "", h.LogEvents.analyticsEvents.NumSaved(), Forced)
h.createEndpointAttemptsMetric(h.LogEvents.Cmd(), h.LogEvents.analyticsEvents.NumFailedAttempts())

// Certificate supportability metrics.
if collector.CertPoolState == collector.SystemCertPoolMissing {
h.Metrics.AddCount("Supportability/PHP/SystemCertificates/Unavailable", "", float64(1), Forced)
Expand All @@ -168,6 +175,7 @@ func (h *Harvest) createFinalMetrics(harvestLimits collector.EventHarvestConfig,
h.Metrics.AddCount("Supportability/EventHarvest/CustomEventData/HarvestLimit", "", float64(harvestLimits.EventConfigs.CustomEventConfig.Limit), Forced)
h.Metrics.AddCount("Supportability/EventHarvest/ErrorEventData/HarvestLimit", "", float64(harvestLimits.EventConfigs.ErrorEventConfig.Limit), Forced)
h.Metrics.AddCount("Supportability/EventHarvest/SpanEventData/HarvestLimit", "", float64(harvestLimits.EventConfigs.SpanEventConfig.Limit), Forced)
h.Metrics.AddCount("Supportability/EventHarvest/LogEventData/HarvestLimit", "", float64(harvestLimits.EventConfigs.LogEventConfig.Limit), Forced)

h.createEndpointAttemptsMetric(h.Metrics.Cmd(), h.Metrics.NumFailedAttempts())

Expand All @@ -194,6 +202,7 @@ func (x *MetricTable) Cmd() string { return collector.CommandMetrics }
func (x *CustomEvents) Cmd() string { return collector.CommandCustomEvents }
func (x *ErrorEvents) Cmd() string { return collector.CommandErrorEvents }
func (x *SpanEvents) Cmd() string { return collector.CommandSpanEvents }
func (x *LogEvents) Cmd() string { return collector.CommandLogEvents }
func (x *ErrorHeap) Cmd() string { return collector.CommandErrors }
func (x *SlowSQLs) Cmd() string { return collector.CommandSlowSQLs }
func (x *TxnTraces) Cmd() string { return collector.CommandTraces }
Expand Down
Loading

0 comments on commit 5546944

Please sign in to comment.