From 46c2bc565e2321d2059c67cdc6304baa86f66328 Mon Sep 17 00:00:00 2001 From: Daniel Odievich Date: Thu, 21 Apr 2022 20:27:25 -0700 Subject: [PATCH] Fixes #13 Added code to filter more non-CSV results returned by SHOW GRANTS TO and SHOW GRANTS ON commands --- .vscode/launch.json | 5 ++-- Helpers/FileIOHelper.cs | 23 +++++++++++++++---- .../Extract/ExtractUsersRolesGrants.cs | 2 +- ProcessingSteps/Index/IndexGrantDetails.cs | 10 ++++---- .../Index/IndexGrantDetailsAccountUsage.cs | 4 ++-- ProcessingSteps/JobStepBase.cs | 10 +++++++- 6 files changed, 39 insertions(+), 15 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index caf449e..8ad7da3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,8 +12,9 @@ // If you have changed target frameworks, make sure to update the program path. "program": "${workspaceFolder}/bin/Debug/net5.0/SFGrantReport.dll", //"args": ["-l", "C:\\snowflake\\SnowflakeGrantReport\\Reports\\aws_cas1", "-r", "C:\\snowflake\\SnowflakeGrantReport\\Reports\\aws_cas1.offline.08192021", "-o", "C:\\snowflake\\SnowflakeGrantReport\\Reports\\aws_cas1.offline.08192021.compare" ], - "args": ["-i", "C:\\snowflake\\SnowflakeGrantReport\\ACCOUNT_USAGE_INPUT\\COINBASE_Snowhouse", "-o", "C:\\snowflake\\SnowflakeGrantReport\\Reports\\COINBASE.20220314"], - //"args": ["-c", "sfgrantreport", "-o", "C:\\snowflake\\SnowflakeGrantReport\\Reports\\Avalera\\myReports", "-s"], + //"args": ["-i", "C:\\snowflake\\SnowflakeGrantReport\\ACCOUNT_USAGE_INPUT\\COINBASE_Snowhouse", "-o", "C:\\snowflake\\SnowflakeGrantReport\\Reports\\COINBASE.20220314"], + "args": ["-c", "aws_cas2.dodievich", "-o", "C:\\snowflake\\SnowflakeGrantReport\\Reports\\aws_cas2.20220421"], + //"args": ["-i", "C:\\snowflake\\SnowflakeGrantReport\\ACCOUNT_USAGE_INPUT\\COINBASE_Snowhouse", "-o", "C:\\snowflake\\SnowflakeGrantReport\\Reports\\COINBASE.20220314"], "cwd": "${workspaceFolder}", // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console "console": "externalTerminal", diff --git a/Helpers/FileIOHelper.cs b/Helpers/FileIOHelper.cs index 0ad0eb4..d936519 100644 --- a/Helpers/FileIOHelper.cs +++ b/Helpers/FileIOHelper.cs @@ -559,10 +559,10 @@ public static MemoryStream WriteListToMemoryStream(List listToWrite, Class public static List ReadListFromCSVFile(string csvFilePath, ClassMap classMap) { - return ReadListFromCSVFile(csvFilePath, classMap, String.Empty); + return ReadListFromCSVFile(csvFilePath, classMap, null); } - public static List ReadListFromCSVFile(string csvFilePath, ClassMap classMap, string skipRecordPrefix) + public static List ReadListFromCSVFile(string csvFilePath, ClassMap classMap, string[] skipRecordValues) { try { @@ -583,9 +583,10 @@ public static List ReadListFromCSVFile(string csvFilePath, ClassMap cla logger.Warn("Bad thing on row {0}, char {1}, field '{2}'", rc.Row, rc.CharPosition, rc.Field); logger.Warn(rc.RawRecord); }; - if (skipRecordPrefix.Length > 0) + if (skipRecordValues != null && skipRecordValues.Length > 0) { - csvReader.Configuration.ShouldSkipRecord = record => record.FirstOrDefault()?.StartsWith(skipRecordPrefix) ?? false; + //csvReader.Configuration.ShouldSkipRecord = record => record.FirstOrDefault()?.StartsWith(skipRecordPrefix) ?? false; + csvReader.Configuration.ShouldSkipRecord = record => ShouldSkipRecordFunction(record, skipRecordValues); } return csvReader.GetRecords().ToList(); } @@ -600,6 +601,20 @@ public static List ReadListFromCSVFile(string csvFilePath, ClassMap cla return null; } + private static bool ShouldSkipRecordFunction(string[] record, string[] skipRecordValues) + { + if (record == null) return true; + + if (record.Length == 0) return true; + + for (int i = 0; i < skipRecordValues.Length; i++) + { + if (record[0].Contains(skipRecordValues[i]) == true) return true; + } + + return false; + } + public static bool AppendTwoCSVFiles(string csvToAppendToFilePath, string csvFromWhichToAppendFilePath) { string folderPath = Path.GetDirectoryName(csvToAppendToFilePath); diff --git a/ProcessingSteps/Extract/ExtractUsersRolesGrants.cs b/ProcessingSteps/Extract/ExtractUsersRolesGrants.cs index 0297686..41e52d6 100644 --- a/ProcessingSteps/Extract/ExtractUsersRolesGrants.cs +++ b/ProcessingSteps/Extract/ExtractUsersRolesGrants.cs @@ -206,7 +206,7 @@ public override bool Execute(ProgramOptions programOptions) #region Future Grants - List grantsToRolesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_RoleShowGrantsTo_FilePath(), new GrantShowGrantsMap(), "No data returned"); + List grantsToRolesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_RoleShowGrantsTo_FilePath(), new GrantShowGrantsMap(), new string[] {"No data returned", "SQL compilation error", "does not exist"}); if (grantsToRolesList != null) { // Get list of all databases and process them diff --git a/ProcessingSteps/Index/IndexGrantDetails.cs b/ProcessingSteps/Index/IndexGrantDetails.cs index cc99a84..723832a 100644 --- a/ProcessingSteps/Index/IndexGrantDetails.cs +++ b/ProcessingSteps/Index/IndexGrantDetails.cs @@ -49,7 +49,7 @@ public override bool Execute(ProgramOptions programOptions) loggerConsole.Info("Process Grants OF"); - List grantsOfRolesAndUsersList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_RoleShowGrantsOf_FilePath(), new RoleMemberShowGrantsMap(), "No data returned"); + List grantsOfRolesAndUsersList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_RoleShowGrantsOf_FilePath(), new RoleMemberShowGrantsMap(), new string[] {"No data returned", "SQL compilation error", "does not exist"}); if (grantsOfRolesAndUsersList != null) { foreach (RoleMember roleMember in grantsOfRolesAndUsersList) @@ -73,28 +73,28 @@ public override bool Execute(ProgramOptions programOptions) List grantsNonUniqueList = new List(); - List grantsOnRolesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_RoleShowGrantsOn_FilePath(), new GrantShowGrantsMap(), "No data returned"); + List grantsOnRolesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_RoleShowGrantsOn_FilePath(), new GrantShowGrantsMap(), new string[] {"No data returned", "SQL compilation error", "does not exist"}); if (grantsOnRolesList != null) { loggerConsole.Info("Granted ON {0} grants", grantsOnRolesList.Count); grantsNonUniqueList.AddRange(grantsOnRolesList); } - List grantsToRolesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_RoleShowGrantsTo_FilePath(), new GrantShowGrantsMap(), "No data returned"); + List grantsToRolesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_RoleShowGrantsTo_FilePath(), new GrantShowGrantsMap(), new string[] {"No data returned", "SQL compilation error", "does not exist"}); if (grantsToRolesList != null) { loggerConsole.Info("Granted TO {0} grants", grantsToRolesList.Count); grantsNonUniqueList.AddRange(grantsToRolesList); } - List grantsFutureDatabasesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_FutureGrantsInDatabases_FilePath(), new GrantShowFutureGrantsMap(), "No data returned"); + List grantsFutureDatabasesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_FutureGrantsInDatabases_FilePath(), new GrantShowFutureGrantsMap(), new string[] {"No data returned", "SQL compilation error", "does not exist"}); if (grantsFutureDatabasesList != null) { loggerConsole.Info("Future Grants on Databases {0} grants", grantsFutureDatabasesList.Count); grantsNonUniqueList.AddRange(grantsFutureDatabasesList); } - List grantsFutureSchemasList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_FutureGrantsInSchemas_FilePath(), new GrantShowFutureGrantsMap(), "No data returned"); + List grantsFutureSchemasList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Data_FutureGrantsInSchemas_FilePath(), new GrantShowFutureGrantsMap(), new string[] {"No data returned", "SQL compilation error", "does not exist"}); if (grantsFutureSchemasList != null) { loggerConsole.Info("Future Grants on Schemas {0} grants", grantsFutureSchemasList.Count); diff --git a/ProcessingSteps/Index/IndexGrantDetailsAccountUsage.cs b/ProcessingSteps/Index/IndexGrantDetailsAccountUsage.cs index e0f964d..0b73ace 100644 --- a/ProcessingSteps/Index/IndexGrantDetailsAccountUsage.cs +++ b/ProcessingSteps/Index/IndexGrantDetailsAccountUsage.cs @@ -54,7 +54,7 @@ public override bool Execute(ProgramOptions programOptions) List grantsOfRolesList = new List(); - List grantsOnRolesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Input_RoleShowGrantsToAndOn_FilePath(), new GrantGrantToRolesMap(), "Initiating login request with your identity provider"); + List grantsOnRolesList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Input_RoleShowGrantsToAndOn_FilePath(), new GrantGrantToRolesMap(), new string[] {"Initiating login request with your identity provider"}); if (grantsOnRolesList != null) { @@ -344,7 +344,7 @@ public override bool Execute(ProgramOptions programOptions) loggerConsole.Info("Process Grants OF Users"); - List grantsOfUsersList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Input_RoleShowGrantsOf_FilePath(), new RoleMemberGrantsToUsersMap(), "Initiating login request with your identity provider"); + List grantsOfUsersList = FileIOHelper.ReadListFromCSVFile(FilePathMap.Input_RoleShowGrantsOf_FilePath(), new RoleMemberGrantsToUsersMap(), new string[] {"Initiating login request with your identity provider"}); if (grantsOfUsersList != null) { foreach (RoleMember roleMember in grantsOfUsersList) diff --git a/ProcessingSteps/JobStepBase.cs b/ProcessingSteps/JobStepBase.cs index df5d9cf..a2260b1 100644 --- a/ProcessingSteps/JobStepBase.cs +++ b/ProcessingSteps/JobStepBase.cs @@ -223,7 +223,8 @@ public static string quoteObjectIdentifier(string objectName) '{', '}', '~', - '`' }; + '`', + '@' }; foreach (var c in roleNameSpecialChars) { // Escape the embedded " @@ -251,6 +252,13 @@ public static string quoteObjectIdentifier(string objectName) { return String.Format("\"{0}\"", objectName); } + + // Finally check for lowercase of the objects. If lowercase exists, quote them + if (objectName.Any(char.IsLower) == true) + { + return String.Format("\"{0}\"", objectName); + } + return objectName; }