Skip to content
This repository has been archived by the owner on Sep 13, 2023. It is now read-only.

Commit

Permalink
Speed improvements for large flat hierarchies
Browse files Browse the repository at this point in the history 
For large flat structure of Roles under single role and many paths up (like single person in each Role)
Some parallelization speedup improvements for the rendering of graphs
  • Loading branch information
@sfc-gh-dodievich
sfc-gh-dodievich committed Mar 5, 2021
1 parent 219a09d commit 7400bdb
Show file tree
Hide file tree
Showing 8 changed files with 357 additions and 203 deletions.
2 changes: 1 addition & 1 deletion .vscode/launch.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// If you have changed target frameworks, make sure to update the program path.
"program": "${workspaceFolder}/bin/Debug/net5.0/SnowGrantReport.dll",
//"args": ["-c", "sfpscogs_dodievich_sso.west-us-2.azure", "-o", "C:\\snowflake\\GrantReport\\Reports\\instacart.carrot.snowhouse"],
"args": ["-i", "C:\\snowflake\\GrantReport\\ACCOUNT_USAGE_INPUT\\INSTACART_CARROT_Snowhouse", "-o", "C:\\snowflake\\GrantReport\\Reports\\instacart.carrot.snowhouse"],
"args": ["-i", "C:\\snowflake\\GrantReport\\ACCOUNT_USAGE_INPUT\\SQUARE_Snowhouse", "-o", "C:\\snowflake\\GrantReport\\Reports\\square.Snowhouse"],
"cwd": "${workspaceFolder}",
// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
"console": "externalTerminal",
Expand Down
94 changes: 83 additions & 11 deletions ProcessingSteps/Index/IndexRoleAndGrantHierarchy.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public override bool Execute(ProgramOptions programOptions)
}

j++;
if (j % 10 == 0)
if (j % 50 == 0)
{
Console.Write("{0}.", j);
}
Expand Down Expand Up @@ -100,10 +100,38 @@ public override bool Execute(ProgramOptions programOptions)

// Load selected types of grants to use to determine whether the role is Functional or Access
// Going to just use SCHEMA and TABLE
List<Grant> grantsToSchemaList = FileIOHelper.ReadListFromCSVFile<Grant>(FilePathMap.Report_RoleGrant_ObjectType_FilePath("SCHEMA"), new GrantMap());
List<Grant> grantsToTableList = FileIOHelper.ReadListFromCSVFile<Grant>(FilePathMap.Report_RoleGrant_ObjectType_FilePath("TABLE"), new GrantMap());
List<Grant> grantsToViewList = FileIOHelper.ReadListFromCSVFile<Grant>(FilePathMap.Report_RoleGrant_ObjectType_FilePath("VIEW"), new GrantMap());
List<Grant> grantsToRoleList = FileIOHelper.ReadListFromCSVFile<Grant>(FilePathMap.Report_RoleGrant_ObjectType_FilePath("ROLE"), new GrantMap());
List<Grant> grantsToSchemaAllList = FileIOHelper.ReadListFromCSVFile<Grant>(FilePathMap.Report_RoleGrant_ObjectType_FilePath("SCHEMA"), new GrantMap());
List<Grant> grantsToTableAllList = FileIOHelper.ReadListFromCSVFile<Grant>(FilePathMap.Report_RoleGrant_ObjectType_FilePath("TABLE"), new GrantMap());
List<Grant> grantsToViewAllList = FileIOHelper.ReadListFromCSVFile<Grant>(FilePathMap.Report_RoleGrant_ObjectType_FilePath("VIEW"), new GrantMap());
List<Grant> grantsToRoleAllList = FileIOHelper.ReadListFromCSVFile<Grant>(FilePathMap.Report_RoleGrant_ObjectType_FilePath("ROLE"), new GrantMap());

Dictionary<string, List<Grant>> grantsToSchemaDict = new Dictionary<string, List<Grant>>();
Dictionary<string, List<Grant>> grantsToTableDict = new Dictionary<string, List<Grant>>();
Dictionary<string, List<Grant>> grantsToViewDict = new Dictionary<string, List<Grant>>();
Dictionary<string, List<Grant>> grantsToRoleDict = new Dictionary<string, List<Grant>>();

var grantsToSchemaGroups = grantsToSchemaAllList.GroupBy(g => g.GrantedTo);
foreach (var grantsGroup in grantsToSchemaGroups)
{
grantsToSchemaDict.Add(grantsGroup.Key, grantsGroup.ToList());
}
var grantsToTableGroups = grantsToTableAllList.GroupBy(g => g.GrantedTo);
foreach (var grantsGroup in grantsToTableGroups)
{
grantsToTableDict.Add(grantsGroup.Key, grantsGroup.ToList());
}
var grantsToViewGroups = grantsToViewAllList.GroupBy(g => g.GrantedTo);
foreach (var grantsGroup in grantsToViewGroups)
{
grantsToViewDict.Add(grantsGroup.Key, grantsGroup.ToList());
}
var grantsToRoleGroups = grantsToRoleAllList.GroupBy(g => g.GrantedTo);
foreach (var grantsGroup in grantsToRoleGroups)
{
grantsToRoleDict.Add(grantsGroup.Key, grantsGroup.ToList());
}

j = 0;

// Detect role types and inheritance rollups
foreach(Role role in rolesList)
Expand Down Expand Up @@ -137,6 +165,14 @@ public override bool Execute(ProgramOptions programOptions)
}

// Check between Functional and Access
List<Grant> grantsToSchemaList = null;
List<Grant> grantsToTableList = null;
List<Grant> grantsToViewList = null;
List<Grant> grantsToRoleList = null;
grantsToSchemaDict.TryGetValue(role.Name, out grantsToSchemaList);
grantsToTableDict.TryGetValue(role.Name, out grantsToTableList);
grantsToViewDict.TryGetValue(role.Name, out grantsToViewList);
grantsToRoleDict.TryGetValue(role.Name, out grantsToRoleList);

// Schemas first
if (role.Type == RoleType.Unknown && grantsToSchemaList != null)
Expand Down Expand Up @@ -170,7 +206,7 @@ public override bool Execute(ProgramOptions programOptions)
// Views third, and only if the role type is still undetermined
if (role.Type == RoleType.Unknown && grantsToViewList != null)
{
List<Grant> grantsToViewForThisRoleList = grantsToTableList.Where(
List<Grant> grantsToViewForThisRoleList = grantsToViewList.Where(
g => g.GrantedTo == role.Name &&
g.Privilege != "USAGE" &&
g.Privilege != "OWNERSHIP" &&
Expand Down Expand Up @@ -213,7 +249,14 @@ public override bool Execute(ProgramOptions programOptions)
}
}
}

j++;
if (j % 50 == 0)
{
Console.Write("{0}.", j);
}
}
Console.WriteLine("Done {0} items", rolesList.Count);

loggerConsole.Info("Building role ancestry paths");

Expand All @@ -225,6 +268,8 @@ public override bool Execute(ProgramOptions programOptions)
{
List<RoleHierarchy> roleHierarchiesList = new List<RoleHierarchy>(grantsToRolesUsageList.Count);

loggerConsole.Info("Processing Role Usage for {0} hierarchy records", grantsToRolesUsageList.Count);

j = 0;

// Build stuff for flow diagrams using the USAGE rights and the role hierarchy
Expand Down Expand Up @@ -291,6 +336,10 @@ public override bool Execute(ProgramOptions programOptions)
}
Console.WriteLine("Done {0} items", grantsToRolesUsageList.Count);

loggerConsole.Info("Looking for stragglers without parents or children for {0} roles", rolesList.Count);

j = 0;

// Now loop through the list of roles looking for the stragglers that have no other roles below them or aren't parented to any
foreach (Role role in rolesList)
{
Expand All @@ -306,8 +355,26 @@ public override bool Execute(ProgramOptions programOptions)

roleHierarchiesList.Add(roleHierarchy);
}

j++;
if (j % 50 == 0)
{
Console.Write("{0}.", j);
}
}

Console.WriteLine("Done {0} items", rolesList.Count);

roleHierarchiesList = roleHierarchiesList.OrderBy(r => r.Name).ThenBy(r => r.GrantedTo).ToList();
FileIOHelper.WriteListToCSVFile<RoleHierarchy>(roleHierarchiesList, new RoleHierarchyMap(), FilePathMap.Report_RoleHierarchy_FilePath());

roleHierarchiesList = null;

GC.Collect();

j = 0;

loggerConsole.Info("Processing Role hierarchy for {0} roles", rolesList.Count);

// For each role, output the hierarchy records that relate to its parents and children
foreach (Role role in rolesList)
{
Expand All @@ -326,8 +393,8 @@ public override bool Execute(ProgramOptions programOptions)

// Get hierarchy of roles
List<RoleHierarchy> thisRoleAndItsRelationsHierarchiesNonUniqueList = new List<RoleHierarchy>(100);
role.GetAllParentRoleHierarchies(role, thisRoleAndItsRelationsHierarchiesNonUniqueList);
role.GetAllChildRoleHierarchies(role, thisRoleAndItsRelationsHierarchiesNonUniqueList);
role.GetAllParentRoleHierarchies(role, thisRoleAndItsRelationsHierarchiesNonUniqueList, 10000);
role.GetAllChildRoleHierarchies(role, thisRoleAndItsRelationsHierarchiesNonUniqueList, 10000);
// Filter to only unique items
var thisRoleAndItsRelationsHierarchiesGrouped = thisRoleAndItsRelationsHierarchiesNonUniqueList.GroupBy(r => String.Format("{0}-{1}", r.Name, r.GrantedTo));
List<RoleHierarchy> thisRoleAndItsRelationsHierarchiesList = new List<RoleHierarchy>(thisRoleAndItsRelationsHierarchiesGrouped.Count());
Expand All @@ -340,10 +407,15 @@ public override bool Execute(ProgramOptions programOptions)

FileIOHelper.WriteListToCSVFile<Role>(thisRoleAndItsRelationsList, new RoleMap(), FilePathMap.Report_RoleDetail_RoleAndItsRelations_FilePath(role.Name));
FileIOHelper.WriteListToCSVFile<RoleHierarchy>(thisRoleAndItsRelationsHierarchiesList, new RoleHierarchyMap(), FilePathMap.Report_RoleHierarchy_RoleAndItsRelations_FilePath(role.Name));

j++;
if (j % 50 == 0)
{
Console.Write("{0}.", j);
}
}
Console.WriteLine("Done {0} items", rolesList.Count);

roleHierarchiesList = roleHierarchiesList.OrderBy(r => r.Name).ThenBy(r => r.GrantedTo).ToList();
FileIOHelper.WriteListToCSVFile<RoleHierarchy>(roleHierarchiesList, new RoleHierarchyMap(), FilePathMap.Report_RoleHierarchy_FilePath());
}
}

Expand Down
2 changes: 1 addition & 1 deletion ProcessingSteps/JobStepBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ public Account buildObjectHierarchyWithGrants()
if (j % 100 == 0)
{
Console.Write("{0}.", j);
}
}
}
Console.WriteLine();
loggerConsole.Info("Done {0} Databases", j);
Expand Down
Loading

0 comments on commit 7400bdb

Please sign in to comment.