diff --git a/.vscode/launch.json b/.vscode/launch.json index ae1ce45..f4e5a70 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,7 +12,7 @@ // If you have changed target frameworks, make sure to update the program path. "program": "${workspaceFolder}/bin/Debug/net5.0/SnowGrantReport.dll", //"args": ["-c", "sfpscogs_dodievich_sso.west-us-2.azure", "-o", "C:\\snowflake\\GrantReport\\Reports\\instacart.carrot.snowhouse"], - "args": ["-i", "C:\\snowflake\\GrantReport\\ACCOUNT_USAGE_INPUT\\INSTACART_CARROT_Snowhouse", "-o", "C:\\snowflake\\GrantReport\\Reports\\instacart.carrot.snowhouse"], + "args": ["-i", "C:\\snowflake\\GrantReport\\ACCOUNT_USAGE_INPUT\\SQUARE_Snowhouse", "-o", "C:\\snowflake\\GrantReport\\Reports\\square.Snowhouse"], "cwd": "${workspaceFolder}", // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console "console": "externalTerminal", diff --git a/ProcessingSteps/Index/IndexRoleAndGrantHierarchy.cs b/ProcessingSteps/Index/IndexRoleAndGrantHierarchy.cs index 04e5481..68f13bf 100644 --- a/ProcessingSteps/Index/IndexRoleAndGrantHierarchy.cs +++ b/ProcessingSteps/Index/IndexRoleAndGrantHierarchy.cs @@ -65,7 +65,7 @@ public override bool Execute(ProgramOptions programOptions) } j++; - if (j % 10 == 0) + if (j % 50 == 0) { Console.Write("{0}.", j); } @@ -100,10 +100,38 @@ public override bool Execute(ProgramOptions programOptions) // Load selected types of grants to use to determine whether the role is Functional or Access // Going to just use SCHEMA and TABLE - List grantsToSchemaList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleGrant_ObjectType_FilePath("SCHEMA"), new GrantMap()); - List grantsToTableList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleGrant_ObjectType_FilePath("TABLE"), new GrantMap()); - List grantsToViewList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleGrant_ObjectType_FilePath("VIEW"), new GrantMap()); - List grantsToRoleList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleGrant_ObjectType_FilePath("ROLE"), new GrantMap()); + List grantsToSchemaAllList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleGrant_ObjectType_FilePath("SCHEMA"), new GrantMap()); + List grantsToTableAllList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleGrant_ObjectType_FilePath("TABLE"), new GrantMap()); + List grantsToViewAllList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleGrant_ObjectType_FilePath("VIEW"), new GrantMap()); + List grantsToRoleAllList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleGrant_ObjectType_FilePath("ROLE"), new GrantMap()); + + Dictionary> grantsToSchemaDict = new Dictionary>(); + Dictionary> grantsToTableDict = new Dictionary>(); + Dictionary> grantsToViewDict = new Dictionary>(); + Dictionary> grantsToRoleDict = new Dictionary>(); + + var grantsToSchemaGroups = grantsToSchemaAllList.GroupBy(g => g.GrantedTo); + foreach (var grantsGroup in grantsToSchemaGroups) + { + grantsToSchemaDict.Add(grantsGroup.Key, grantsGroup.ToList()); + } + var grantsToTableGroups = grantsToTableAllList.GroupBy(g => g.GrantedTo); + foreach (var grantsGroup in grantsToTableGroups) + { + grantsToTableDict.Add(grantsGroup.Key, grantsGroup.ToList()); + } + var grantsToViewGroups = grantsToViewAllList.GroupBy(g => g.GrantedTo); + foreach (var grantsGroup in grantsToViewGroups) + { + grantsToViewDict.Add(grantsGroup.Key, grantsGroup.ToList()); + } + var grantsToRoleGroups = grantsToRoleAllList.GroupBy(g => g.GrantedTo); + foreach (var grantsGroup in grantsToRoleGroups) + { + grantsToRoleDict.Add(grantsGroup.Key, grantsGroup.ToList()); + } + + j = 0; // Detect role types and inheritance rollups foreach(Role role in rolesList) @@ -137,6 +165,14 @@ public override bool Execute(ProgramOptions programOptions) } // Check between Functional and Access + List grantsToSchemaList = null; + List grantsToTableList = null; + List grantsToViewList = null; + List grantsToRoleList = null; + grantsToSchemaDict.TryGetValue(role.Name, out grantsToSchemaList); + grantsToTableDict.TryGetValue(role.Name, out grantsToTableList); + grantsToViewDict.TryGetValue(role.Name, out grantsToViewList); + grantsToRoleDict.TryGetValue(role.Name, out grantsToRoleList); // Schemas first if (role.Type == RoleType.Unknown && grantsToSchemaList != null) @@ -170,7 +206,7 @@ public override bool Execute(ProgramOptions programOptions) // Views third, and only if the role type is still undetermined if (role.Type == RoleType.Unknown && grantsToViewList != null) { - List grantsToViewForThisRoleList = grantsToTableList.Where( + List grantsToViewForThisRoleList = grantsToViewList.Where( g => g.GrantedTo == role.Name && g.Privilege != "USAGE" && g.Privilege != "OWNERSHIP" && @@ -213,7 +249,14 @@ public override bool Execute(ProgramOptions programOptions) } } } + + j++; + if (j % 50 == 0) + { + Console.Write("{0}.", j); + } } + Console.WriteLine("Done {0} items", rolesList.Count); loggerConsole.Info("Building role ancestry paths"); @@ -225,6 +268,8 @@ public override bool Execute(ProgramOptions programOptions) { List roleHierarchiesList = new List(grantsToRolesUsageList.Count); + loggerConsole.Info("Processing Role Usage for {0} hierarchy records", grantsToRolesUsageList.Count); + j = 0; // Build stuff for flow diagrams using the USAGE rights and the role hierarchy @@ -291,6 +336,10 @@ public override bool Execute(ProgramOptions programOptions) } Console.WriteLine("Done {0} items", grantsToRolesUsageList.Count); + loggerConsole.Info("Looking for stragglers without parents or children for {0} roles", rolesList.Count); + + j = 0; + // Now loop through the list of roles looking for the stragglers that have no other roles below them or aren't parented to any foreach (Role role in rolesList) { @@ -306,8 +355,26 @@ public override bool Execute(ProgramOptions programOptions) roleHierarchiesList.Add(roleHierarchy); } + + j++; + if (j % 50 == 0) + { + Console.Write("{0}.", j); + } } - + Console.WriteLine("Done {0} items", rolesList.Count); + + roleHierarchiesList = roleHierarchiesList.OrderBy(r => r.Name).ThenBy(r => r.GrantedTo).ToList(); + FileIOHelper.WriteListToCSVFile(roleHierarchiesList, new RoleHierarchyMap(), FilePathMap.Report_RoleHierarchy_FilePath()); + + roleHierarchiesList = null; + + GC.Collect(); + + j = 0; + + loggerConsole.Info("Processing Role hierarchy for {0} roles", rolesList.Count); + // For each role, output the hierarchy records that relate to its parents and children foreach (Role role in rolesList) { @@ -326,8 +393,8 @@ public override bool Execute(ProgramOptions programOptions) // Get hierarchy of roles List thisRoleAndItsRelationsHierarchiesNonUniqueList = new List(100); - role.GetAllParentRoleHierarchies(role, thisRoleAndItsRelationsHierarchiesNonUniqueList); - role.GetAllChildRoleHierarchies(role, thisRoleAndItsRelationsHierarchiesNonUniqueList); + role.GetAllParentRoleHierarchies(role, thisRoleAndItsRelationsHierarchiesNonUniqueList, 10000); + role.GetAllChildRoleHierarchies(role, thisRoleAndItsRelationsHierarchiesNonUniqueList, 10000); // Filter to only unique items var thisRoleAndItsRelationsHierarchiesGrouped = thisRoleAndItsRelationsHierarchiesNonUniqueList.GroupBy(r => String.Format("{0}-{1}", r.Name, r.GrantedTo)); List thisRoleAndItsRelationsHierarchiesList = new List(thisRoleAndItsRelationsHierarchiesGrouped.Count()); @@ -340,10 +407,15 @@ public override bool Execute(ProgramOptions programOptions) FileIOHelper.WriteListToCSVFile(thisRoleAndItsRelationsList, new RoleMap(), FilePathMap.Report_RoleDetail_RoleAndItsRelations_FilePath(role.Name)); FileIOHelper.WriteListToCSVFile(thisRoleAndItsRelationsHierarchiesList, new RoleHierarchyMap(), FilePathMap.Report_RoleHierarchy_RoleAndItsRelations_FilePath(role.Name)); + + j++; + if (j % 50 == 0) + { + Console.Write("{0}.", j); + } } + Console.WriteLine("Done {0} items", rolesList.Count); - roleHierarchiesList = roleHierarchiesList.OrderBy(r => r.Name).ThenBy(r => r.GrantedTo).ToList(); - FileIOHelper.WriteListToCSVFile(roleHierarchiesList, new RoleHierarchyMap(), FilePathMap.Report_RoleHierarchy_FilePath()); } } diff --git a/ProcessingSteps/JobStepBase.cs b/ProcessingSteps/JobStepBase.cs index a261bda..28dd393 100644 --- a/ProcessingSteps/JobStepBase.cs +++ b/ProcessingSteps/JobStepBase.cs @@ -191,7 +191,7 @@ public Account buildObjectHierarchyWithGrants() if (j % 100 == 0) { Console.Write("{0}.", j); - } + } } Console.WriteLine(); loggerConsole.Info("Done {0} Databases", j); diff --git a/ProcessingSteps/Report/ReportUserRoleGrantHierarchyGraphViz.cs b/ProcessingSteps/Report/ReportUserRoleGrantHierarchyGraphViz.cs index 600fdcc..82b5b95 100644 --- a/ProcessingSteps/Report/ReportUserRoleGrantHierarchyGraphViz.cs +++ b/ProcessingSteps/Report/ReportUserRoleGrantHierarchyGraphViz.cs @@ -4,6 +4,8 @@ using System.Linq; using System.Net; using System.Text; +using System.Threading; +using System.Threading.Tasks; using System.Xml; using Snowflake.GrantReport.ReportObjects; @@ -51,213 +53,236 @@ public override bool Execute(ProgramOptions programOptions) Role syntheticRoleAll = new Role(); syntheticRoleAll.Name = "ALL_ROLES_TOGETHER_SYNTHETIC"; rolesList.Insert(0, syntheticRoleAll); - - // Create a graphviz file for each role - foreach (Role role in rolesList) + + ParallelOptions parallelOptions = new ParallelOptions(); + if (programOptions.ProcessSequentially == true) { - loggerConsole.Info("Processing visualization for {0}", role); + parallelOptions.MaxDegreeOfParallelism = 1; + } - List thisRoleAndItsRelationsHierarchiesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleHierarchy_RoleAndItsRelations_FilePath(role.Name), new RoleHierarchyMap());; - List thisRoleAndItsRelationsList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleDetail_RoleAndItsRelations_FilePath(role.Name), new RoleMap()); + int j = 0; - if (role == syntheticRoleAll) + Parallel.ForEach( + rolesList, + parallelOptions, + () => 0, + (role, loop, subtotal) => { - thisRoleAndItsRelationsHierarchiesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleHierarchy_FilePath(), new RoleHierarchyMap());; - thisRoleAndItsRelationsList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleDetail_FilePath(), new RoleMap()); - } + logger.Info("Processing visualization for {0}", role); - if (thisRoleAndItsRelationsList != null && thisRoleAndItsRelationsHierarchiesList != null) - { - Dictionary rolesDict = thisRoleAndItsRelationsList.ToDictionary(k => k.Name, r => r); - Dictionary roleNamesOutput = new Dictionary(thisRoleAndItsRelationsList.Count); - Role roleBeingOutput = null; - - StringBuilder sbGraphViz = new StringBuilder(64 * thisRoleAndItsRelationsHierarchiesList.Count + 128); - - // Start the graph and set its default settings - sbGraphViz.AppendLine("digraph {"); - sbGraphViz.AppendLine(" layout=\"dot\";"); - sbGraphViz.AppendLine(" rankdir=\"TB\";"); - sbGraphViz.AppendLine(" center=true;"); - sbGraphViz.AppendLine(" splines=\"ortho\";"); - sbGraphViz.AppendLine(" overlap=false;"); - //sbGraphViz.AppendLine(" colorscheme=\"SVG\";"); - sbGraphViz.AppendLine(" node [shape=\"rect\" style=\"filled,rounded\" fontname=\"Courier New\"];"); - sbGraphViz.AppendLine(" edge [fontname=\"Courier New\"];"); - - sbGraphViz.AppendFormat(" // Graph for the Role {0}", role); sbGraphViz.AppendLine(); - - #region Role boxes - - // Role boxes - sbGraphViz.AppendLine(); - sbGraphViz.AppendLine(" // Roles"); - sbGraphViz.AppendLine (" subgraph cluster_roles {"); - sbGraphViz.AppendFormat(" label = \"roles related to: {0}\";", role); sbGraphViz.AppendLine(); - foreach (RoleHierarchy roleHierarchy in thisRoleAndItsRelationsHierarchiesList) + List thisRoleAndItsRelationsHierarchiesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleHierarchy_RoleAndItsRelations_FilePath(role.Name), new RoleHierarchyMap());; + List thisRoleAndItsRelationsList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleDetail_RoleAndItsRelations_FilePath(role.Name), new RoleMap()); + + if (role == syntheticRoleAll) { - if (roleHierarchy.GrantedTo != "" && roleNamesOutput.ContainsKey(roleHierarchy.GrantedTo) == false) - { - // Name of the role with color - rolesDict.TryGetValue(roleHierarchy.GrantedTo, out roleBeingOutput); - sbGraphViz.AppendFormat(" \"{0}\"{1};", roleHierarchy.GrantedTo.Replace("\"", "\\\""), getRoleStyleAttribute(roleBeingOutput)); sbGraphViz.AppendLine(); - roleNamesOutput.Add(roleHierarchy.GrantedTo, roleBeingOutput); - } - - if (roleNamesOutput.ContainsKey(roleHierarchy.Name) == false) - { - // Name of the role with color - rolesDict.TryGetValue(roleHierarchy.Name, out roleBeingOutput); - sbGraphViz.AppendFormat(" \"{0}\"{1};", roleHierarchy.Name.Replace("\"", "\\\""), getRoleStyleAttribute(roleBeingOutput)); sbGraphViz.AppendLine(); - roleNamesOutput.Add(roleHierarchy.Name, roleBeingOutput); - } + thisRoleAndItsRelationsHierarchiesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleHierarchy_FilePath(), new RoleHierarchyMap());; + thisRoleAndItsRelationsList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Report_RoleDetail_FilePath(), new RoleMap()); } - sbGraphViz.AppendLine(" }// /Roles"); - #endregion - - #region Role hierachy - - // Role connections - sbGraphViz.AppendLine(); - sbGraphViz.AppendLine(" // Role hierarchy"); - foreach (RoleHierarchy roleHierarchy in thisRoleAndItsRelationsHierarchiesList) + if (thisRoleAndItsRelationsList != null && thisRoleAndItsRelationsHierarchiesList != null) { - if (roleHierarchy.GrantedTo == "") continue; + Dictionary rolesDict = thisRoleAndItsRelationsList.ToDictionary(k => k.Name, r => r); + Dictionary roleNamesOutput = new Dictionary(thisRoleAndItsRelationsList.Count); + Role roleBeingOutput = null; - // Role to role connector - sbGraphViz.AppendFormat(" \"{0}\"->\"{1}\";", roleHierarchy.GrantedTo.Replace("\"", "\\\""), roleHierarchy.Name.Replace("\"", "\\\"")); sbGraphViz.AppendLine(); - } - sbGraphViz.AppendLine(" // /Role hierarchy"); + StringBuilder sbGraphViz = new StringBuilder(64 * thisRoleAndItsRelationsHierarchiesList.Count + 128); - #endregion + // Start the graph and set its default settings + sbGraphViz.AppendLine("digraph {"); + sbGraphViz.AppendLine(" layout=\"dot\";"); + sbGraphViz.AppendLine(" rankdir=\"TB\";"); + sbGraphViz.AppendLine(" center=true;"); + sbGraphViz.AppendLine(" splines=\"ortho\";"); + sbGraphViz.AppendLine(" overlap=false;"); + //sbGraphViz.AppendLine(" colorscheme=\"SVG\";"); + sbGraphViz.AppendLine(" node [shape=\"rect\" style=\"filled,rounded\" fontname=\"Courier New\"];"); + sbGraphViz.AppendLine(" edge [fontname=\"Courier New\"];"); - if (role != syntheticRoleAll) - { - #region Databases, Schemas, Tables and Views + sbGraphViz.AppendFormat(" // Graph for the Role {0}", role); sbGraphViz.AppendLine(); + + #region Role boxes + // Role boxes sbGraphViz.AppendLine(); - sbGraphViz.AppendLine(" // Databases"); - sbGraphViz.AppendLine(" subgraph cluster_db_wrapper {"); - sbGraphViz.AppendLine(" label = \"Databases\";"); + sbGraphViz.AppendLine(" // Roles"); + sbGraphViz.AppendLine (" subgraph cluster_roles {"); + sbGraphViz.AppendFormat(" label = \"roles related to: {0}\";", role); sbGraphViz.AppendLine(); + foreach (RoleHierarchy roleHierarchy in thisRoleAndItsRelationsHierarchiesList) + { + if (roleHierarchy.GrantedTo != "" && roleNamesOutput.ContainsKey(roleHierarchy.GrantedTo) == false) + { + // Name of the role with color + rolesDict.TryGetValue(roleHierarchy.GrantedTo, out roleBeingOutput); + sbGraphViz.AppendFormat(" \"{0}\"{1};", roleHierarchy.GrantedTo.Replace("\"", "\\\""), getRoleStyleAttribute(roleBeingOutput)); sbGraphViz.AppendLine(); + roleNamesOutput.Add(roleHierarchy.GrantedTo, roleBeingOutput); + } + + if (roleNamesOutput.ContainsKey(roleHierarchy.Name) == false) + { + // Name of the role with color + rolesDict.TryGetValue(roleHierarchy.Name, out roleBeingOutput); + sbGraphViz.AppendFormat(" \"{0}\"{1};", roleHierarchy.Name.Replace("\"", "\\\""), getRoleStyleAttribute(roleBeingOutput)); sbGraphViz.AppendLine(); + roleNamesOutput.Add(roleHierarchy.Name, roleBeingOutput); + } + } + sbGraphViz.AppendLine(" }// /Roles"); + + #endregion + + #region Role hierachy + + // Role connections sbGraphViz.AppendLine(); + sbGraphViz.AppendLine(" // Role hierarchy"); + foreach (RoleHierarchy roleHierarchy in thisRoleAndItsRelationsHierarchiesList) + { + if (roleHierarchy.GrantedTo == "") continue; - int databaseIndex = 0; - foreach (Database database in account.Databases) + // Role to role connector + sbGraphViz.AppendFormat(" \"{0}\"->\"{1}\";", roleHierarchy.GrantedTo.Replace("\"", "\\\""), roleHierarchy.Name.Replace("\"", "\\\"")); sbGraphViz.AppendLine(); + } + sbGraphViz.AppendLine(" // /Role hierarchy"); + + #endregion + + if (role != syntheticRoleAll) { - // Should output database - bool isDatabaseRelatedToSelectedRole = false; - foreach (Grant grant in database.Grants) + #region Databases, Schemas, Tables and Views + + sbGraphViz.AppendLine(); + sbGraphViz.AppendLine(" // Databases"); + sbGraphViz.AppendLine(" subgraph cluster_db_wrapper {"); + sbGraphViz.AppendLine(" label = \"Databases\";"); + sbGraphViz.AppendLine(); + + int databaseIndex = 0; + foreach (Database database in account.Databases) { - if (grant.Privilege == "USAGE" || grant.Privilege == "OWNERSHIP") + // Should output database + bool isDatabaseRelatedToSelectedRole = false; + foreach (Grant grant in database.Grants) { - if (roleNamesOutput.ContainsKey(grant.GrantedTo) == true) + if (grant.Privilege == "USAGE" || grant.Privilege == "OWNERSHIP") { - isDatabaseRelatedToSelectedRole = true; - break; + if (roleNamesOutput.ContainsKey(grant.GrantedTo) == true) + { + isDatabaseRelatedToSelectedRole = true; + break; + } } } - } - if (isDatabaseRelatedToSelectedRole == false) continue; - - // Output database - sbGraphViz.AppendFormat(" // Database {0}", database.FullName); sbGraphViz.AppendLine(); - sbGraphViz.AppendFormat(" subgraph cluster_db_{0} {{", databaseIndex); sbGraphViz.AppendLine(); - sbGraphViz.AppendLine (" style=\"filled\";"); - sbGraphViz.AppendLine (" fillcolor=\"snow\";"); - sbGraphViz.AppendFormat(" label = \"db: {0}\";", database.ShortName); sbGraphViz.AppendLine(); - sbGraphViz.AppendLine (" node [shape=\"cylinder\" fillcolor=\"darkkhaki\"];"); - sbGraphViz.AppendLine(); - - sbGraphViz.AppendFormat(" \"{0}\";", database.FullName); sbGraphViz.AppendLine(); - sbGraphViz.AppendLine(); - - // List of schemas with number of tables and views - sbGraphViz.AppendFormat(" \"{0}.schema\" [shape=\"folder\" label=<", database.FullName); sbGraphViz.AppendLine(); - sbGraphViz.AppendLine (" "); - sbGraphViz.AppendLine (" "); - - int schemaLimit = 0; - foreach (Schema schema in database.Schemas) - { - // Only output - if (schemaLimit >= 10) + if (isDatabaseRelatedToSelectedRole == false) continue; + + // Output database + sbGraphViz.AppendFormat(" // Database {0}", database.FullName); sbGraphViz.AppendLine(); + sbGraphViz.AppendFormat(" subgraph cluster_db_{0} {{", databaseIndex); sbGraphViz.AppendLine(); + sbGraphViz.AppendLine (" style=\"filled\";"); + sbGraphViz.AppendLine (" fillcolor=\"snow\";"); + sbGraphViz.AppendFormat(" label = \"db: {0}\";", database.ShortName); sbGraphViz.AppendLine(); + sbGraphViz.AppendLine (" node [shape=\"cylinder\" fillcolor=\"darkkhaki\"];"); + sbGraphViz.AppendLine(); + + sbGraphViz.AppendFormat(" \"{0}\";", database.FullName); sbGraphViz.AppendLine(); + sbGraphViz.AppendLine(); + + // List of schemas with number of tables and views + sbGraphViz.AppendFormat(" \"{0}.schema\" [shape=\"folder\" label=<", database.FullName); sbGraphViz.AppendLine(); + sbGraphViz.AppendLine ("
STV
"); + sbGraphViz.AppendLine (" "); + + int schemaLimit = 0; + foreach (Schema schema in database.Schemas) { - sbGraphViz.AppendFormat(" ", database.Schemas.Count); sbGraphViz.AppendLine(); + // Only output + if (schemaLimit >= 10) + { + sbGraphViz.AppendFormat(" ", database.Schemas.Count); sbGraphViz.AppendLine(); - break; - } + break; + } - sbGraphViz.AppendFormat(" ", schema.ShortName, schema.Tables.Count, schema.Views.Count); sbGraphViz.AppendLine(); + sbGraphViz.AppendFormat(" ", schema.ShortName, schema.Tables.Count, schema.Views.Count); sbGraphViz.AppendLine(); - schemaLimit++; - } - sbGraphViz.AppendLine ("
STV
Up to {0}......
Up to {0}......
{0}{1}{2}
{0}{1}{2}
>];"); + schemaLimit++; + } + sbGraphViz.AppendLine (" >];"); - // Connect database to schemas - sbGraphViz.AppendFormat(" \"{0}\"->\"{0}.schema\" [style=\"invis\"];", database.FullName); sbGraphViz.AppendLine(); + // Connect database to schemas + sbGraphViz.AppendFormat(" \"{0}\"->\"{0}.schema\" [style=\"invis\"];", database.FullName); sbGraphViz.AppendLine(); - sbGraphViz.AppendFormat(" }} // /Database {0}", database.FullName); sbGraphViz.AppendLine(); + sbGraphViz.AppendFormat(" }} // /Database {0}", database.FullName); sbGraphViz.AppendLine(); - databaseIndex++; - } + databaseIndex++; + } - sbGraphViz.AppendLine(" } // /Databases"); + sbGraphViz.AppendLine(" } // /Databases"); - #endregion + #endregion - #region Roles using databases - - sbGraphViz.AppendLine(); - sbGraphViz.AppendLine(" // Roles using databases"); + #region Roles using databases + + sbGraphViz.AppendLine(); + sbGraphViz.AppendLine(" // Roles using databases"); - // Output connectors from roles USAGE'ing databases - foreach (Database database in account.Databases) - { - foreach (Grant grant in database.Grants) + // Output connectors from roles USAGE'ing databases + foreach (Database database in account.Databases) { - if (grant.Privilege == "USAGE" || grant.Privilege == "OWNERSHIP") + foreach (Grant grant in database.Grants) { - if (roleNamesOutput.ContainsKey(grant.GrantedTo) == true) + if (grant.Privilege == "USAGE" || grant.Privilege == "OWNERSHIP") { - sbGraphViz.AppendFormat(" \"{0}\"->\"{1}\" [color=\"darkkhaki\"];", grant.GrantedTo, grant.ObjectNameUnquoted); sbGraphViz.AppendLine(); + if (roleNamesOutput.ContainsKey(grant.GrantedTo) == true) + { + sbGraphViz.AppendFormat(" \"{0}\"->\"{1}\" [color=\"darkkhaki\"];", grant.GrantedTo, grant.ObjectNameUnquoted); sbGraphViz.AppendLine(); + } } } } + + sbGraphViz.AppendLine(" // /Roles using databases"); + + #endregion } - sbGraphViz.AppendLine(" // /Roles using databases"); + #region Legend + // Output Legend + sbGraphViz.AppendLine(); + string legend = @" // Legend + ""legend"" [label=< + + + + + + + + + + +
Legend
BUILT IN
SCIM
ROLE MANAGEMENT
FUNCTIONAL
FUNCTIONAL NOT UNDER SYSADMIN
ACCESS
ACCESS NOT UNDER SYSADMIN
NOT UNDER ACCOUNTADMIN
>];"; + sbGraphViz.AppendLine(legend); + #endregion - } - #region Legend - - // Output Legend - sbGraphViz.AppendLine(); - string legend = @" // Legend - ""legend"" [label=< - - - - - - - - - - -
Legend
BUILT IN
SCIM
ROLE MANAGEMENT
FUNCTIONAL
FUNCTIONAL NOT UNDER SYSADMIN
ACCESS
ACCESS NOT UNDER SYSADMIN
NOT UNDER ACCOUNTADMIN
>];"; - sbGraphViz.AppendLine(legend); - - #endregion + // Close the graph + sbGraphViz.AppendLine("}"); + + FileIOHelper.SaveFileToPath(sbGraphViz.ToString(), FilePathMap.Report_GraphViz_RoleAndItsRelationsGrants_FilePath(role.Name), false); + } - // Close the graph - sbGraphViz.AppendLine("}"); - - FileIOHelper.SaveFileToPath(sbGraphViz.ToString(), FilePathMap.Report_GraphViz_RoleAndItsRelationsGrants_FilePath(role.Name), false); + return 1; + }, + (finalResult) => + { + Interlocked.Add(ref j, finalResult); + if (j % 50 == 0) + { + Console.Write("[{0}].", j); + } } - } + ); + loggerConsole.Info("Completed {0} Roles", rolesList.Count); #endregion @@ -379,25 +404,44 @@ public override bool Execute(ProgramOptions programOptions) if (graphVizDriver.ExecutableFilePath.Length > 0) { - foreach (Role role in rolesList) - { - graphVizDriver.ConvertGraphVizToFile( - FilePathMap.Report_GraphViz_RoleAndItsRelationsGrants_FilePath(role.Name), - FilePathMap.Report_Diagram_SVG_RoleAndItsRelationsGrants_FilePath(role.Name, true), - "svg"); - graphVizDriver.ConvertGraphVizToFile( - FilePathMap.Report_GraphViz_RoleAndItsRelationsGrants_FilePath(role.Name), - FilePathMap.Report_Diagram_PNG_RoleAndItsRelationsGrants_FilePath(role.Name, true), - "png"); - graphVizDriver.ConvertGraphVizToFile( - FilePathMap.Report_GraphViz_RoleAndItsRelationsGrants_FilePath(role.Name), - FilePathMap.Report_Diagram_PDF_RoleAndItsRelationsGrants_FilePath(role.Name, true), - "pdf"); - } + j = 0; + + Parallel.ForEach( + rolesList, + parallelOptions, + () => 0, + (role, loop, subtotal) => + { + loggerConsole.Info("Rendering graphs for {0}", role); + + graphVizDriver.ConvertGraphVizToFile( + FilePathMap.Report_GraphViz_RoleAndItsRelationsGrants_FilePath(role.Name), + FilePathMap.Report_Diagram_SVG_RoleAndItsRelationsGrants_FilePath(role.Name, true), + "svg"); + graphVizDriver.ConvertGraphVizToFile( + FilePathMap.Report_GraphViz_RoleAndItsRelationsGrants_FilePath(role.Name), + FilePathMap.Report_Diagram_PNG_RoleAndItsRelationsGrants_FilePath(role.Name, true), + "png"); + graphVizDriver.ConvertGraphVizToFile( + FilePathMap.Report_GraphViz_RoleAndItsRelationsGrants_FilePath(role.Name), + FilePathMap.Report_Diagram_PDF_RoleAndItsRelationsGrants_FilePath(role.Name, true), + "pdf"); + + return 1; + }, + (finalResult) => + { + Interlocked.Add(ref j, finalResult); + if (j % 10 == 0) + { + Console.Write("[{0}].", j); + } + } + ); + loggerConsole.Info("Completed {0} Roles", rolesList.Count); } #endregion - } return true; diff --git a/ProgramOptions.cs b/ProgramOptions.cs index ba16bd5..dac10ff 100644 --- a/ProgramOptions.cs +++ b/ProgramOptions.cs @@ -17,6 +17,9 @@ public class ProgramOptions [Option('d', "delete-previous-report-output", Required = false, HelpText = "If true, delete any results of previous processing.")] public bool DeletePreviousReportOutput { get; set; } + [Option('s', "sequential-processing", Required = false, HelpText = "If true, process certain items during extraction and conversion sequentially.")] + public bool ProcessSequentially { get; set; } + public string ReportJobFilePath { get; set; } public ReportJob ReportJob { get; set; } public string ProgramLocationFolderPath { get; set; } @@ -24,11 +27,12 @@ public class ProgramOptions public override string ToString() { return String.Format( - "ProgramOptions: ConnectionName='{0}' OutputFolderPath='{1}' DeletePreviousReportOutput='{2}' InputFolderPath='{3}'", + "ProgramOptions: ConnectionName='{0}' OutputFolderPath='{1}' DeletePreviousReportOutput='{2}' InputFolderPath='{3}' ProcessSequentially='{4}'", this.ConnectionName, this.ReportFolderPath, this.DeletePreviousReportOutput, - this.InputFolderPath); + this.InputFolderPath, + this.ProcessSequentially); } } diff --git a/ReportObjects/Role/Role.cs b/ReportObjects/Role/Role.cs index 5e4bd06..bcac65e 100644 --- a/ReportObjects/Role/Role.cs +++ b/ReportObjects/Role/Role.cs @@ -95,7 +95,7 @@ public string AncestryPaths get { List allPaths = new List(); - GetParentPaths(this, this.Name, allPaths); + GetParentPaths(this, this.Name, allPaths, 25); string[] ancestryPathsArray = allPaths.Select(p => p.ToString()).ToArray(); this.NumAncestryPaths = ancestryPathsArray.Length; return String.Join('\n', ancestryPathsArray); @@ -146,7 +146,7 @@ private bool doesRoleRollupToRole(Role thisRole, Role potentialParent) } } - public void GetParentPaths(Role thisRole, string pathSoFar, List allPaths) + public void GetParentPaths(Role thisRole, string pathSoFar, List allPaths, int maxPathsToReturn) { if (thisRole.ParentRoles.Count == 0) { @@ -157,12 +157,38 @@ public void GetParentPaths(Role thisRole, string pathSoFar, List allPath foreach (Role parentRole in thisRole.ParentRoles) { string pathSoFar1 = String.Format(@"{1}->{0}", pathSoFar, parentRole.Name); - GetParentPaths(parentRole, pathSoFar1, allPaths); + GetParentPaths(parentRole, pathSoFar1, allPaths, maxPathsToReturn); + if (allPaths.Count >= maxPathsToReturn) + { + return; + } } return; } } + // public void GetParentPaths(Role thisRole, StringBuilder pathSoFar, List allPaths, int maxPathsToReturn) + // { + // if (thisRole.ParentRoles.Count == 0) + // { + // allPaths.Add(pathSoFar.ToString()); + // } + // else + // { + // foreach (Role parentRole in thisRole.ParentRoles) + // { + // pathSoFar.Insert(0, String.Format("{0}->", parentRole.Name)); + // //string pathSoFar1 = String.Format(@"{1}->{0}", pathSoFar, parentRole.Name); + // GetParentPaths(parentRole, pathSoFar, allPaths, maxPathsToReturn); + // if (allPaths.Count >= maxPathsToReturn) + // { + // return; + // } + // } + // return; + // } + // } + public void GetAllParentRoles(Role thisRole, List listOfRoles) { if (thisRole.ParentRoles.Count == 0) @@ -180,7 +206,7 @@ public void GetAllParentRoles(Role thisRole, List listOfRoles) } } - public void GetAllParentRoleHierarchies(Role thisRole, List listOfRoleHierarchies) + public void GetAllParentRoleHierarchies(Role thisRole, List listOfRoleHierarchies, int maxHierarchies) { if (thisRole.ParentRoles.Count == 0) { @@ -198,7 +224,11 @@ public void GetAllParentRoleHierarchies(Role thisRole, List listO roleHierarchy.NumAncestryPaths = roleHierarchy.AncestryPaths.Split('\n').Count(); roleHierarchy.DirectAncestry = String.Format("{0}->{1}", roleHierarchy.GrantedTo, roleHierarchy.Name); listOfRoleHierarchies.Add(roleHierarchy); - GetAllParentRoleHierarchies(parentRole, listOfRoleHierarchies); + if (listOfRoleHierarchies.Count >= maxHierarchies) + { + return; + } + GetAllParentRoleHierarchies(parentRole, listOfRoleHierarchies, maxHierarchies); } return; } @@ -221,7 +251,7 @@ public void GetAllChildRoles(Role thisRole, List listOfRoles) } } - public void GetAllChildRoleHierarchies(Role thisRole, List listOfRoleHierarchies) + public void GetAllChildRoleHierarchies(Role thisRole, List listOfRoleHierarchies, int maxHierarchies) { if (thisRole.ChildRoles.Count == 0) { @@ -239,7 +269,11 @@ public void GetAllChildRoleHierarchies(Role thisRole, List listOf roleHierarchy.NumAncestryPaths = roleHierarchy.AncestryPaths.Split('\n').Count(); roleHierarchy.DirectAncestry = String.Format("{0}->{1}", roleHierarchy.GrantedTo, roleHierarchy.Name); listOfRoleHierarchies.Add(roleHierarchy); - GetAllChildRoleHierarchies(childRole, listOfRoleHierarchies); + if (listOfRoleHierarchies.Count >= maxHierarchies) + { + return; + } + GetAllChildRoleHierarchies(childRole, listOfRoleHierarchies, maxHierarchies); } return; } diff --git a/Snowflake.GrantReport.csproj b/Snowflake.GrantReport.csproj index 2e11397..8aba9bf 100644 --- a/Snowflake.GrantReport.csproj +++ b/Snowflake.GrantReport.csproj @@ -7,9 +7,9 @@ SnowGrantReport Snowflake.GrantReport - 2021.2.23.0 - 2021.2.23.0 - 2021.2.23.0 + 2021.3.4.0 + 2021.3.4.0 + 2021.3.4.0 Daniel Odievich (daniel.odievich@snowflake.com) Snowflake Computing Snowflake Grant Report diff --git a/ZipReleases.ps1 b/ZipReleases.ps1 index 94a4252..1a91970 100644 --- a/ZipReleases.ps1 +++ b/ZipReleases.ps1 @@ -1,6 +1,6 @@ $zip = "C:\Program Files\7-Zip\7z.exe" -$version = "2021.2.23.0" +$version = "2021.3.4.0" cd "C:\snowflake\snowgrantreport\bin\Publish\win" & $zip a "C:\snowflake\GrantReport\Releases\$version\SnowGrantReport.win.$version.zip" '@C:\snowflake\snowgrantreport\Release\listfile.win.txt'