From 416d8a8b8db9eb11854051f78691a5835fbbeb96 Mon Sep 17 00:00:00 2001 From: Lukas Kohl Date: Fri, 20 Sep 2024 15:44:06 +0200 Subject: [PATCH 1/5] RSP File Support for Linux Makes support for .rsp files cross-platform --- src/BinSkim.Driver/BinSkim.cs | 45 ++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/src/BinSkim.Driver/BinSkim.cs b/src/BinSkim.Driver/BinSkim.cs index 70310a72..8ebe6c07 100644 --- a/src/BinSkim.Driver/BinSkim.cs +++ b/src/BinSkim.Driver/BinSkim.cs @@ -16,7 +16,7 @@ internal static class BinSkim { private static int Main(string[] args) { - args = EntryPointUtilities.GenerateArguments(args, new FileSystem(), new EnvironmentVariables()); + args = GenerateArguments(args, new FileSystem(), new EnvironmentVariables()); args = RewriteArgs(args); var rewrittenArgs = new List(args); @@ -96,5 +96,48 @@ private static bool EvaluatesToTrueOrFalse(string value) value == "true" || value == "false" || value == "1" || value == "0"; } + + public static string[] GenerateArguments( + string[] args, + IFileSystem fileSystem, + IEnvironmentVariables environmentVariables) + { + var expandedArguments = new List(); + + foreach (string argument in args) + { + if (!IsResponseFileArgument(argument)) + { + expandedArguments.Add(argument); + continue; + } + + string responseFile = argument.Trim('"').Substring(1); + + responseFile = environmentVariables.ExpandEnvironmentVariables(responseFile); + responseFile = fileSystem.PathGetFullPath(responseFile); + + string[] responseFileLines = fileSystem.FileReadAllLines(responseFile); + ExpandResponseFile(responseFileLines, expandedArguments); + } + + return expandedArguments.ToArray(); + } + + private static bool IsResponseFileArgument(string argument) + { + return argument.Length > 1 && argument[0] == '@'; + } + + private static void ExpandResponseFile(string[] responseFileLines, List expandedArguments) + { + foreach (string responseFileLine in responseFileLines) + { + List fileList = ArgumentSplitter.CommandLineToArgvW(responseFileLine.Trim()) ?? + throw new InvalidOperationException("Could not parse response file line:" + responseFileLine); + + expandedArguments.AddRange(fileList); + } + } } } From 918713cb09055d356e8e2ba4afa526561bf3b923 Mon Sep 17 00:00:00 2001 From: Lukas Kohl Date: Tue, 8 Oct 2024 14:29:43 +0200 Subject: [PATCH 2/5] Argument & RSP support file segmentation --- src/BinSkim.Driver/BinSkim.cs | 45 +-------------------- src/BinSkim.Driver/ExpandArguments.cs | 57 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 44 deletions(-) create mode 100644 src/BinSkim.Driver/ExpandArguments.cs diff --git a/src/BinSkim.Driver/BinSkim.cs b/src/BinSkim.Driver/BinSkim.cs index 8ebe6c07..91671476 100644 --- a/src/BinSkim.Driver/BinSkim.cs +++ b/src/BinSkim.Driver/BinSkim.cs @@ -16,7 +16,7 @@ internal static class BinSkim { private static int Main(string[] args) { - args = GenerateArguments(args, new FileSystem(), new EnvironmentVariables()); + args = ExpandArguments.GenerateArguments(args, new FileSystem(), new EnvironmentVariables()); args = RewriteArgs(args); var rewrittenArgs = new List(args); @@ -96,48 +96,5 @@ private static bool EvaluatesToTrueOrFalse(string value) value == "true" || value == "false" || value == "1" || value == "0"; } - - public static string[] GenerateArguments( - string[] args, - IFileSystem fileSystem, - IEnvironmentVariables environmentVariables) - { - var expandedArguments = new List(); - - foreach (string argument in args) - { - if (!IsResponseFileArgument(argument)) - { - expandedArguments.Add(argument); - continue; - } - - string responseFile = argument.Trim('"').Substring(1); - - responseFile = environmentVariables.ExpandEnvironmentVariables(responseFile); - responseFile = fileSystem.PathGetFullPath(responseFile); - - string[] responseFileLines = fileSystem.FileReadAllLines(responseFile); - ExpandResponseFile(responseFileLines, expandedArguments); - } - - return expandedArguments.ToArray(); - } - - private static bool IsResponseFileArgument(string argument) - { - return argument.Length > 1 && argument[0] == '@'; - } - - private static void ExpandResponseFile(string[] responseFileLines, List expandedArguments) - { - foreach (string responseFileLine in responseFileLines) - { - List fileList = ArgumentSplitter.CommandLineToArgvW(responseFileLine.Trim()) ?? - throw new InvalidOperationException("Could not parse response file line:" + responseFileLine); - - expandedArguments.AddRange(fileList); - } - } } } diff --git a/src/BinSkim.Driver/ExpandArguments.cs b/src/BinSkim.Driver/ExpandArguments.cs new file mode 100644 index 00000000..dad95da0 --- /dev/null +++ b/src/BinSkim.Driver/ExpandArguments.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; + +using Microsoft.CodeAnalysis.Sarif; +using Microsoft.CodeAnalysis.Sarif.Driver; + +namespace Microsoft.CodeAnalysis.IL +{ + public static class ExpandArguments + { + public static string[] GenerateArguments( + string[] args, + IFileSystem fileSystem, + IEnvironmentVariables environmentVariables) + { + var expandedArguments = new List(); + + foreach (string argument in args) + { + if (!IsResponseFileArgument(argument)) + { + expandedArguments.Add(argument); + continue; + } + + string responseFile = argument.Trim('"').Substring(1); + + responseFile = environmentVariables.ExpandEnvironmentVariables(responseFile); + responseFile = fileSystem.PathGetFullPath(responseFile); + + string[] responseFileLines = fileSystem.FileReadAllLines(responseFile); + ExpandResponseFile(responseFileLines, expandedArguments); + } + + return expandedArguments.ToArray(); + } + + private static bool IsResponseFileArgument(string argument) + { + return argument.Length > 1 && argument[0] == '@'; + } + + private static void ExpandResponseFile(string[] responseFileLines, List expandedArguments) + { + foreach (string responseFileLine in responseFileLines) + { + List fileList = ArgumentSplitter.CommandLineToArgvW(responseFileLine.Trim()) ?? + throw new InvalidOperationException("Could not parse response file line:" + responseFileLine); + + expandedArguments.AddRange(fileList); + } + } + } +} From 656a9f2092427f1932f97977545f60b279d7fea7 Mon Sep 17 00:00:00 2001 From: Lukas Kohl Date: Tue, 8 Oct 2024 14:30:15 +0200 Subject: [PATCH 3/5] Argument Splitting and RSP Support Tests --- .../ExpandArgumentsUnitTests.cs | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 src/Test.UnitTests.BinSkim.Driver/ExpandArgumentsUnitTests.cs diff --git a/src/Test.UnitTests.BinSkim.Driver/ExpandArgumentsUnitTests.cs b/src/Test.UnitTests.BinSkim.Driver/ExpandArgumentsUnitTests.cs new file mode 100644 index 00000000..d516466f --- /dev/null +++ b/src/Test.UnitTests.BinSkim.Driver/ExpandArgumentsUnitTests.cs @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Globalization; +using System.IO; + +using FluentAssertions; + +using Microsoft.CodeAnalysis.IL; + +using Microsoft.CodeAnalysis.Sarif; +using Microsoft.CodeAnalysis.Sarif.Driver; + +using Moq; + +using Xunit; + + +namespace Microsoft.CodeAnalysis.BinSkim.Rules +{ + public class ExpandArgumentsUnitTests + { + private static void SetupTestMocks( + string responseFileName, + string[] responseFileContents, + out Mock fileSystemMock, + out Mock environmentVariablesMock) + { + fileSystemMock = new Mock(); + environmentVariablesMock = new Mock(); + + fileSystemMock.Setup(fs => fs.PathGetFullPath(responseFileName)).Returns(responseFileName); + fileSystemMock.Setup(fs => fs.FileReadAllLines(responseFileName)).Returns(responseFileContents); + environmentVariablesMock.Setup(ev => ev.ExpandEnvironmentVariables(responseFileName)).Returns(responseFileName); + } + + [Fact] + public void GenerateArguments_SucceedsWithEmptyArgumentList() + { + string[] result = ExpandArguments.GenerateArguments(Array.Empty(), null, null); + + result.Should().BeEmpty(); + } + + [Fact] + public void GenerateArguments_SucceedsWithNormalArguments() + { + string[] args = new[] { "/y:z", "/x" }; + + string[] result = ExpandArguments.GenerateArguments(args, null, null); + + result.Length.Should().Be(2); + result.Should().ContainInOrder(args); + } + + [Fact] + public void GenerateArguments_ExceptionIfResponseFileDoesNotExist() + { + string NonexistentResponseFile = Guid.NewGuid().ToString() + ".rsp"; + string[] args = new[] { "/a", "@" + NonexistentResponseFile, "/f" }; + + Assert.Throws( + () => ExpandArguments.GenerateArguments(args, new FileSystem(), new EnvironmentVariables()) + ); + } + + [Theory] + [InlineData(new[] { "/b", "/c:val /d", " /e " }, new[] { "/a", "/b", "/c:val", "/d", "/e", "/f" })] + public void GenerateArguments_ExpandsResponseFileContents(string[] rspContent, string[] expected) + { + const string ResponseFileName = "Mocked.rsp"; + string[] args = new[] { "/a", "@" + ResponseFileName, "/f" }; + + SetupTestMocks( + ResponseFileName, + rspContent, + out Mock fileSystemMock, + out Mock environmentVariablesMock); + + IFileSystem fileSystem = fileSystemMock.Object; + IEnvironmentVariables environmentVariables = environmentVariablesMock.Object; + + string[] result = ExpandArguments.GenerateArguments(args, fileSystem, environmentVariables); + + result.Should().ContainInOrder(expected); + + fileSystemMock.Verify(fs => fs.PathGetFullPath(ResponseFileName), Times.Once); + fileSystemMock.Verify(fs => fs.FileReadAllLines(ResponseFileName), Times.Once); + environmentVariablesMock.Verify(ev => ev.ExpandEnvironmentVariables(ResponseFileName), Times.Once); + } + + [Theory] + [InlineData(new[] { "a \"one two\" b" }, new[] { "a", "one two", "b" })] + public void GenerateArguments_StripsQuotesFromAroundArgsWithSpacesInResponseFiles(string[] rspContent, string[] expected) + { + const string ResponseFileName = "Mocked.rsp"; + string[] args = new[] { "@" + ResponseFileName }; + + SetupTestMocks( + ResponseFileName, + rspContent, + out Mock fileSystemMock, + out Mock environmentVariablesMock); + + IFileSystem fileSystem = fileSystemMock.Object; + IEnvironmentVariables environmentVariables = environmentVariablesMock.Object; + + string[] result = ExpandArguments.GenerateArguments(args, fileSystem, environmentVariables); + + result.Length.Should().Be(3); + result.Should().ContainInOrder(expected); + + fileSystemMock.Verify(fs => fs.PathGetFullPath(ResponseFileName), Times.Once); + fileSystemMock.Verify(fs => fs.FileReadAllLines(ResponseFileName), Times.Once); + environmentVariablesMock.Verify(ev => ev.ExpandEnvironmentVariables(ResponseFileName), Times.Once); + } + + [Theory] + [InlineData(new[] { "a \"one two\" b" }, new[] { "a", "one two", "b" })] + public void GenerateArguments_ExpandsEnvironmentVariablesInResponseFilePathName(string[] rspContent, string[] expected) + { + const string DirectoryVariableName = "InstallationDirectory"; + const string ResponseFileName = "Mocked.rsp"; + + string responseFileNameArgument = string.Format( + CultureInfo.InvariantCulture, + @"%{0}%\{1}", + DirectoryVariableName, + ResponseFileName + ); + + string[] args = new[] { "@" + responseFileNameArgument }; + + SetupTestMocks( + responseFileNameArgument, + rspContent, + out Mock fileSystemMock, + out Mock environmentVariablesMock); + + IFileSystem fileSystem = fileSystemMock.Object; + IEnvironmentVariables environmentVariables = environmentVariablesMock.Object; + + string[] result = ExpandArguments.GenerateArguments(args, fileSystem, environmentVariables); + + result.Length.Should().Be(3); + result.Should().ContainInOrder(expected); + + fileSystemMock.Verify(fs => fs.PathGetFullPath(responseFileNameArgument), Times.Once); + fileSystemMock.Verify(fs => fs.FileReadAllLines(responseFileNameArgument), Times.Once); + environmentVariablesMock.Verify(ev => ev.ExpandEnvironmentVariables(responseFileNameArgument), Times.Once); + } + } +} From b1b5d8a0c3360c2c108b4bf18a19455341f19901 Mon Sep 17 00:00:00 2001 From: Lukas Kohl Date: Thu, 17 Oct 2024 14:54:00 +0200 Subject: [PATCH 4/5] Correctly Trim Escaped Arguments --- src/BinSkim.Driver/ExpandArguments.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/BinSkim.Driver/ExpandArguments.cs b/src/BinSkim.Driver/ExpandArguments.cs index dad95da0..0dea5395 100644 --- a/src/BinSkim.Driver/ExpandArguments.cs +++ b/src/BinSkim.Driver/ExpandArguments.cs @@ -20,13 +20,14 @@ public static string[] GenerateArguments( foreach (string argument in args) { - if (!IsResponseFileArgument(argument)) + string trimArgument = argument.Trim('"'); + if (!IsResponseFileArgument(trimArgument)) { - expandedArguments.Add(argument); + expandedArguments.Add(trimArgument); continue; } - string responseFile = argument.Trim('"').Substring(1); + string responseFile = trimArgument.Substring(1); responseFile = environmentVariables.ExpandEnvironmentVariables(responseFile); responseFile = fileSystem.PathGetFullPath(responseFile); From b39e16acb4f021c54b387497e7b11340e8766006 Mon Sep 17 00:00:00 2001 From: Lukas Kohl Date: Thu, 17 Oct 2024 14:57:59 +0200 Subject: [PATCH 5/5] Ignore Comments in .rsp files Example of an .rsp file: "/example/to/binary/example.exe" # Binary # ---- Libraries ---- "/example/to/libraries/library1.dll" "/example/to/libraries/library2.dll" would result in: "/example/to/binary/example.exe" "/example/to/libraries/library1.dll" "/example/to/libraries/library2.dll" --- src/BinSkim.Driver/ExpandArguments.cs | 11 +++++++- .../ExpandArgumentsUnitTests.cs | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/BinSkim.Driver/ExpandArguments.cs b/src/BinSkim.Driver/ExpandArguments.cs index 0dea5395..f1cf5d7b 100644 --- a/src/BinSkim.Driver/ExpandArguments.cs +++ b/src/BinSkim.Driver/ExpandArguments.cs @@ -48,7 +48,16 @@ private static void ExpandResponseFile(string[] responseFileLines, List { foreach (string responseFileLine in responseFileLines) { - List fileList = ArgumentSplitter.CommandLineToArgvW(responseFileLine.Trim()) ?? + string responseFilePath = responseFileLine; + + // Ignore comments from response file lines + int commentIndex = responseFileLine.IndexOf('#'); + if (commentIndex >= 0) + { + responseFilePath = responseFileLine.Substring(0, commentIndex); + } + + List fileList = ArgumentSplitter.CommandLineToArgvW(responseFilePath.Trim()) ?? throw new InvalidOperationException("Could not parse response file line:" + responseFileLine); expandedArguments.AddRange(fileList); diff --git a/src/Test.UnitTests.BinSkim.Driver/ExpandArgumentsUnitTests.cs b/src/Test.UnitTests.BinSkim.Driver/ExpandArgumentsUnitTests.cs index d516466f..e328541d 100644 --- a/src/Test.UnitTests.BinSkim.Driver/ExpandArgumentsUnitTests.cs +++ b/src/Test.UnitTests.BinSkim.Driver/ExpandArgumentsUnitTests.cs @@ -90,6 +90,32 @@ public void GenerateArguments_ExpandsResponseFileContents(string[] rspContent, s environmentVariablesMock.Verify(ev => ev.ExpandEnvironmentVariables(ResponseFileName), Times.Once); } + [Theory] + [InlineData(new[] { "/b", "/c:val /d", "# Random Comment", " /e " }, new[] { "/a", "/b", "/c:val", "/d", "/e", "/f" })] + [InlineData(new[] { "/b", "/c:val /d#Another Comment", " /e " }, new[] { "/a", "/b", "/c:val", "/d", "/e", "/f" })] + public void GenerateArguments_TrimCommentsFromResponseFileContents(string[] rspContent, string[] expected) + { + const string ResponseFileName = "Mocked.rsp"; + string[] args = new[] { "/a", "@" + ResponseFileName, "/f" }; + + SetupTestMocks( + ResponseFileName, + rspContent, + out Mock fileSystemMock, + out Mock environmentVariablesMock); + + IFileSystem fileSystem = fileSystemMock.Object; + IEnvironmentVariables environmentVariables = environmentVariablesMock.Object; + + string[] result = ExpandArguments.GenerateArguments(args, fileSystem, environmentVariables); + + result.Should().ContainInOrder(expected); + + fileSystemMock.Verify(fs => fs.PathGetFullPath(ResponseFileName), Times.Once); + fileSystemMock.Verify(fs => fs.FileReadAllLines(ResponseFileName), Times.Once); + environmentVariablesMock.Verify(ev => ev.ExpandEnvironmentVariables(ResponseFileName), Times.Once); + } + [Theory] [InlineData(new[] { "a \"one two\" b" }, new[] { "a", "one two", "b" })] public void GenerateArguments_StripsQuotesFromAroundArgsWithSpacesInResponseFiles(string[] rspContent, string[] expected)