Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

#95891 Microdot: grainReq: optionally log Orleans system grains #247

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

#95891 Microdot: grainReq: optionally log Orleans system grains #247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 72 additions & 45 deletions Gigya.Microdot.Orleans.Hosting/GigyaSiloHost.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,15 @@
#endregion

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using Gigya.Common.Contracts.Exceptions;
using Gigya.Microdot.Hosting.HttpService;
using Gigya.Microdot.Interfaces.Configuration;
using Gigya.Microdot.Interfaces.Events;
using Gigya.Microdot.Interfaces.Logging;
using Gigya.Microdot.Orleans.Hosting.Events;
Expand Down Expand Up @@ -58,18 +60,29 @@ public class GigyaSiloHost
private ISourceBlock<OrleansConfig> OrleansConfigSourceBlock { get; }
public static OrleansConfig PreviousOrleansConfig { get; private set; }
private Func<LoadShedding> LoadSheddingConfig { get; }
private Func<GrainLogging> GrainLoggingConfig { get; }

[ConfigurationRoot("Microdot.GrainLogging", RootStrategy.ReplaceClassNameWithPath)]
public class GrainLogging : IConfigObject
{
public bool LogServiceGrains;
public bool LogMicrodotGrains;
public bool LogOrleansGrains;
}



public GigyaSiloHost(ILog log, OrleansConfigurationBuilder configBuilder,
HttpServiceListener httpServiceListener,
IEventPublisher<GrainCallEvent> eventPublisher, Func<LoadShedding> loadSheddingConfig,
ISourceBlock<OrleansConfig> orleansConfigSourceBlock, OrleansConfig orleansConfig)
ISourceBlock<OrleansConfig> orleansConfigSourceBlock, OrleansConfig orleansConfig, Func<GrainLogging> grainLoggingConfig)
{
Log = log;
ConfigBuilder = configBuilder;
HttpServiceListener = httpServiceListener;
EventPublisher = eventPublisher;
LoadSheddingConfig = loadSheddingConfig;
GrainLoggingConfig = grainLoggingConfig;


OrleansConfigSourceBlock = orleansConfigSourceBlock;
Expand Down Expand Up @@ -178,47 +191,6 @@ private async Task BootstrapInit(IProviderRuntime providerRuntime)
public TaskScheduler GrainTaskScheduler { get; set; }


private void OutgoingCallInterceptor(InvokeMethodRequest request, IGrain target)
{
TracingContext.SetUpStorage();
TracingContext.SpanStartTime = DateTimeOffset.UtcNow;
}


private async Task<object> IncomingCallInterceptor(MethodInfo targetMethod, InvokeMethodRequest request, IGrain target, IGrainMethodInvoker invoker)
{
if (targetMethod == null)
throw new ArgumentNullException(nameof(targetMethod));

var declaringNameSpace = targetMethod.DeclaringType?.Namespace;

// Do not intercept Orleans grains or other grains which should not be included in statistics.
if (targetMethod.DeclaringType.GetCustomAttribute<ExcludeGrainFromStatisticsAttribute>() != null ||
declaringNameSpace?.StartsWith("Orleans") == true)
return await invoker.Invoke(target, request);

RequestTimings.GetOrCreate(); // Ensure request timings is created here and not in the grain call.

RequestTimings.Current.Request.Start();
Exception ex = null;

try
{
RejectRequestIfLateOrOverloaded();
return await invoker.Invoke(target, request);
}
catch (Exception e)
{
ex = e;
throw;
}
finally
{
RequestTimings.Current.Request.Stop();
PublishEvent(targetMethod, target, ex);
}
}

public void UpdateOrleansAgeLimitChange(OrleansConfig orleanConfig)
{
try
Expand Down Expand Up @@ -256,13 +228,68 @@ public void UpdateOrleansAgeLimitChange(OrleansConfig orleanConfig)
}
}

private void RejectRequestIfLateOrOverloaded()

private void OutgoingCallInterceptor(InvokeMethodRequest request, IGrain target)
{
TracingContext.SetUpStorage();
TracingContext.SpanStartTime = DateTimeOffset.UtcNow;
}


static ConcurrentDictionary<Type, bool> MICRODOT_GRAINS = new ConcurrentDictionary<Type, bool>();

private async Task<object> IncomingCallInterceptor(MethodInfo targetMethod, InvokeMethodRequest request, IGrain target, IGrainMethodInvoker invoker)
{
if (targetMethod == null)
throw new ArgumentNullException(nameof(targetMethod));

// Identify the grain type
bool isOrleansGrain = targetMethod.DeclaringType?.Namespace?.StartsWith("Orleans") == true;
bool isMicrodotGrain = MICRODOT_GRAINS.GetOrAdd(targetMethod.DeclaringType, type => type.GetCustomAttribute<ExcludeGrainFromStatisticsAttribute>() != null);
bool isServiceGrain = !isOrleansGrain && !isMicrodotGrain;

// Drop the request if we're overloaded
var loadSheddingConfig = LoadSheddingConfig();
if ( (loadSheddingConfig.ApplyToOrleansGrains && isOrleansGrain)
|| (loadSheddingConfig.ApplyToMicrodotGrains && isMicrodotGrain)
|| (loadSheddingConfig.ApplyToServiceGrains && isServiceGrain))
{
RejectRequestIfLateOrOverloaded(loadSheddingConfig);
}

RequestTimings.GetOrCreate(); // Ensure request timings is created here and not in the grain call.
RequestTimings.Current.Request.Start();
Exception ex = null;

try
{
return await invoker.Invoke(target, request);
}
catch (Exception e)
{
ex = e;
throw;
}
finally
{
RequestTimings.Current.Request.Stop();
var loggingConfig = GrainLoggingConfig();
if ( (loggingConfig.LogOrleansGrains && isOrleansGrain)
|| (loggingConfig.LogMicrodotGrains && isMicrodotGrain)
|| (loggingConfig.LogServiceGrains && isServiceGrain))
{
PublishEvent(targetMethod, target, ex);
}
}
}


private void RejectRequestIfLateOrOverloaded(LoadShedding config)
{
var config = LoadSheddingConfig();
var now = DateTimeOffset.UtcNow;

// Too much time passed since our direct caller made the request to us; something's causing a delay. Log or reject the request, if needed.
if (config.DropOrleansRequestsBySpanTime != LoadShedding.Toggle.Disabled
if ( config.DropOrleansRequestsBySpanTime != LoadShedding.Toggle.Disabled
&& TracingContext.SpanStartTime != null
&& TracingContext.SpanStartTime.Value + config.DropOrleansRequestsOlderThanSpanTimeBy < now)
{
Expand Down
4 changes: 4 additions & 0 deletions Gigya.Microdot.SharedLogic/Configurations/LoadShedding.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ public enum Toggle

public Toggle DropOrleansRequestsBySpanTime { get; set; } = Toggle.Disabled;
public TimeSpan DropOrleansRequestsOlderThanSpanTimeBy { get; set; } = TimeSpan.FromSeconds(5);

public bool ApplyToServiceGrains { get; set; } = true;
public bool ApplyToMicrodotGrains { get; set; } = false;
public bool ApplyToOrleansGrains { get; set; } = false;
}

}
6 changes: 4 additions & 2 deletions ...ya.Microdot.Orleans.Hosting.UnitTests/Microservice/WarmupTestService/GigyaSiloHostFake.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@ namespace Gigya.Microdot.Orleans.Hosting.UnitTests.Microservice.WarmupTestServic
{
public class GigyaSiloHostFake : GigyaSiloHost
{
public GigyaSiloHostFake(WarmupTestServiceHostWithSiloHostFake host, ILog log, OrleansConfigurationBuilder configBuilder, HttpServiceListener httpServiceListener, IEventPublisher<GrainCallEvent> eventPublisher, Func<LoadShedding> loadSheddingConfig, ISourceBlock<OrleansConfig> orleansConfigSourceBlock, OrleansConfig orleansConfig) :
base(log, configBuilder, httpServiceListener, eventPublisher, loadSheddingConfig, orleansConfigSourceBlock, orleansConfig)
public GigyaSiloHostFake(WarmupTestServiceHostWithSiloHostFake host, ILog log, OrleansConfigurationBuilder configBuilder,
HttpServiceListener httpServiceListener, IEventPublisher<GrainCallEvent> eventPublisher, Func<LoadShedding> loadSheddingConfig,
ISourceBlock<OrleansConfig> orleansConfigSourceBlock, OrleansConfig orleansConfig, Func<GrainLogging> grainLoggingConfig) :
base(log, configBuilder, httpServiceListener, eventPublisher, loadSheddingConfig, orleansConfigSourceBlock, orleansConfig, grainLoggingConfig)
{
try
{
Expand Down