From 445141255c0de628ec3dd199fbbb3100292a4eeb Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Wed, 17 Jan 2024 15:37:13 +0100 Subject: [PATCH 1/3] Fixed handling unicode characters in headings --- .../Helpers/MarkdownExtensions.cs | 20 ++++++++ .../DocLinkChecker.Test/HyperlinkTests.cs | 43 +++++++++++++++++ .../DocLinkChecker.Test/MarkdownTests.cs | 46 ++++++++++++++++++- .../DocLinkChecker/Helpers/MarkdownHelper.cs | 4 +- .../Properties/launchSettings.json | 8 ---- 5 files changed, 111 insertions(+), 10 deletions(-) delete mode 100644 src/DocLinkChecker/DocLinkChecker/Properties/launchSettings.json diff --git a/src/DocLinkChecker/DocLinkChecker.Test/Helpers/MarkdownExtensions.cs b/src/DocLinkChecker/DocLinkChecker.Test/Helpers/MarkdownExtensions.cs index 4ca6ab4..a332894 100644 --- a/src/DocLinkChecker/DocLinkChecker.Test/Helpers/MarkdownExtensions.cs +++ b/src/DocLinkChecker/DocLinkChecker.Test/Helpers/MarkdownExtensions.cs @@ -48,6 +48,17 @@ internal static string AddLink(this string s, string url) return s + Environment.NewLine + content; } + internal static string AddCodeLink(this string s, string name, string url) + { + Faker faker = new Faker(); + string content = $" [!code-csharp[{name}]({url})]" + Environment.NewLine; + if (string.IsNullOrEmpty(s)) + { + return content; + } + return s + Environment.NewLine + content; + } + internal static string AddTableStart(this string s, int columns = 3) { Faker faker = new Faker(); @@ -102,5 +113,14 @@ internal static string AddRawMarkdown(this string s, string markdown) } return s + markdown; } + + internal static string AddRawContent(this string s, string content) + { + if (string.IsNullOrEmpty(s)) + { + return content; + } + return s + content; + } } } diff --git a/src/DocLinkChecker/DocLinkChecker.Test/HyperlinkTests.cs b/src/DocLinkChecker/DocLinkChecker.Test/HyperlinkTests.cs index 57ab102..eb9d389 100644 --- a/src/DocLinkChecker/DocLinkChecker.Test/HyperlinkTests.cs +++ b/src/DocLinkChecker/DocLinkChecker.Test/HyperlinkTests.cs @@ -295,6 +295,49 @@ public async void ValidateLocalLinkWithFullPathShouldHaveErrors() service.Errors.First().Message.Should().Contain("Full path not allowed"); } + [Fact] + public async void ValidateCodeReferenceFile() + { + // Arrange + _config.DocumentationFiles.SourceFolder = _fileServiceMock.Root; + _config.DocLinkChecker.RelativeLinkStrategy = RelativeLinkType.All; + + string foo = "scripts/foo.cs"; + string fooPath = Path.GetFullPath(Path.Combine(_fileServiceMock.Root, foo)); + string empty = string.Empty; + _fileServiceMock.Files.Add(fooPath, empty.AddRawContent( +@"// (other content above) + +// +foreach(var thing in list) +{ + // Do Stuff +} +// +")); + string bar = "scripts/bar.cs"; + string barPath = Path.GetFullPath(Path.Combine(_fileServiceMock.Root, bar)); + _fileServiceMock.Files.Add(barPath, empty.AddRawContent( +@"private void MyMethod() +{ + #region starting + // Some code + #endregion +} +")); + + LinkValidatorService service = new LinkValidatorService(_serviceProvider, _config, _fileService, _console); + + //Act + int line = 124; + int column = 3381; + Hyperlink link = new Hyperlink($"{_fileServiceMock.Root}\\index.md", line, column, "[!code-csharp[Code](scripts/foo.cs)]"); + await service.VerifyHyperlink(link); + + // Assert + service.Errors.Where(x => x.Severity == MarkdownErrorSeverity.Error).Should().BeEmpty(); + } + private ICustomConsoleLogger GetMockedConsoleLogger() { Mock console = new Mock(); diff --git a/src/DocLinkChecker/DocLinkChecker.Test/MarkdownTests.cs b/src/DocLinkChecker/DocLinkChecker.Test/MarkdownTests.cs index 85837c7..13e9960 100644 --- a/src/DocLinkChecker/DocLinkChecker.Test/MarkdownTests.cs +++ b/src/DocLinkChecker/DocLinkChecker.Test/MarkdownTests.cs @@ -1,6 +1,7 @@ -namespace DocLinkChecker.Test +namespace DocLinkChecker.Test { using System.Linq; + using Bogus; using DocLinkChecker.Helpers; using DocLinkChecker.Models; using DocLinkChecker.Test.Helpers; @@ -53,6 +54,17 @@ public class MarkdownTests .AddNewLine() .AddParagraphs(2); + private string _codeLinkingDocument = string.Empty + .AddHeading("Code Linking Document", 1) + .AddParagraphs(1).AddCodeLink("Code", "scripts/foo.cs") + .AddParagraphs(1).AddCodeLink("Code", "scripts/foo.cs?name=Bravo") + .AddNewLine() + .AddParagraphs(1).AddCodeLink(string.Empty, "scripts/bar.cs#starting") + .AddParagraphs(1).AddCodeLink(string.Empty, "scripts/bar.cs#L3-5") + .AddParagraphs(1).AddCodeLink(string.Empty, "scripts/bar.cs?highlight=2,5-7,9-)") + .AddNewLine() + .AddParagraphs(2); + [Fact] public void FindAllLinks() { @@ -81,6 +93,31 @@ public void FindAllHeadings() headings.Count.Should().Be(6); } + [Fact] + public void FindAllHeadingsWithUnicodeCharacters() + { + string markdown = string.Empty + .AddHeading("Test Unicode Characters", 1) + .AddParagraphs(1).AddLink("#") + .AddHeading("abcdefghijklmnopqrstuvwxyz 0123456789", 2) + .AddParagraphs(1) + .AddHeading("ABCDEFGHIJKLMNOPQRSTUVWXYZ 0123456789", 2) + .AddParagraphs(1) + .AddHeading("UNICODE-!@#$%^&*+=~`<>,.?/:;€|Æäßéóčúįǯ-CHARS", 2) + .AddParagraphs(1); + + var result = MarkdownHelper.ParseMarkdownString(string.Empty, markdown, true); + + var headings = result.objects + .OfType() + .ToList(); + + headings.Count.Should().Be(4); + headings[1].Id.Should().Be("abcdefghijklmnopqrstuvwxyz-0123456789"); + headings[2].Id.Should().Be("abcdefghijklmnopqrstuvwxyz-0123456789"); + headings[3].Id.Should().Be("unicode-æäßéóčúįǯ-chars"); + } + [Fact] public void FindAllTables() { @@ -94,5 +131,12 @@ public void FindAllTablesWithErrors() var result = MarkdownHelper.ParseMarkdownString(string.Empty, _errorDocument, true); result.errors.Count.Should().Be(5); } + + [Fact] + public void FindAllCodeLinks() + { + var result = MarkdownHelper.ParseMarkdownString(string.Empty, _codeLinkingDocument, false); + result.errors.Should().BeEmpty(); + } } } \ No newline at end of file diff --git a/src/DocLinkChecker/DocLinkChecker/Helpers/MarkdownHelper.cs b/src/DocLinkChecker/DocLinkChecker/Helpers/MarkdownHelper.cs index 9abee37..1979c30 100644 --- a/src/DocLinkChecker/DocLinkChecker/Helpers/MarkdownHelper.cs +++ b/src/DocLinkChecker/DocLinkChecker/Helpers/MarkdownHelper.cs @@ -80,7 +80,9 @@ public static (List objects, List errors) // custom generation of the id string id = title.ToLower(); id = Regex.Replace(id, "[ _]", "-"); - id = Regex.Replace(id, "[^a-zA-Z0-9-]*", string.Empty); + + // replace all non-characters. \p[L] takes all unicode variants in account as well like ö and á + id = Regex.Replace(id, @"[^\p{L}0-9-]*", string.Empty); return new Heading(markdownFilePath, x.Line + 1, x.Column + 1, title, id); }) diff --git a/src/DocLinkChecker/DocLinkChecker/Properties/launchSettings.json b/src/DocLinkChecker/DocLinkChecker/Properties/launchSettings.json deleted file mode 100644 index 84dd06f..0000000 --- a/src/DocLinkChecker/DocLinkChecker/Properties/launchSettings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "profiles": { - "DocLinkChecker": { - "commandName": "Project", - "commandLineArgs": "-d \"D:\\Git\\CSE\\LSEG\\LSEG-ef-doc\\docs\" -tavx -f \"D:\\Git\\CSE\\ZF\\DMPwork\\docfx-companion-tools.json\"" - } - } -} \ No newline at end of file From ae2c7563837663a4b1307e6709f2cd602ad0de74 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Wed, 17 Jan 2024 15:50:31 +0100 Subject: [PATCH 2/3] Removed test for investigating another issue --- .../DocLinkChecker.Test/HyperlinkTests.cs | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/src/DocLinkChecker/DocLinkChecker.Test/HyperlinkTests.cs b/src/DocLinkChecker/DocLinkChecker.Test/HyperlinkTests.cs index eb9d389..57ab102 100644 --- a/src/DocLinkChecker/DocLinkChecker.Test/HyperlinkTests.cs +++ b/src/DocLinkChecker/DocLinkChecker.Test/HyperlinkTests.cs @@ -295,49 +295,6 @@ public async void ValidateLocalLinkWithFullPathShouldHaveErrors() service.Errors.First().Message.Should().Contain("Full path not allowed"); } - [Fact] - public async void ValidateCodeReferenceFile() - { - // Arrange - _config.DocumentationFiles.SourceFolder = _fileServiceMock.Root; - _config.DocLinkChecker.RelativeLinkStrategy = RelativeLinkType.All; - - string foo = "scripts/foo.cs"; - string fooPath = Path.GetFullPath(Path.Combine(_fileServiceMock.Root, foo)); - string empty = string.Empty; - _fileServiceMock.Files.Add(fooPath, empty.AddRawContent( -@"// (other content above) - -// -foreach(var thing in list) -{ - // Do Stuff -} -// -")); - string bar = "scripts/bar.cs"; - string barPath = Path.GetFullPath(Path.Combine(_fileServiceMock.Root, bar)); - _fileServiceMock.Files.Add(barPath, empty.AddRawContent( -@"private void MyMethod() -{ - #region starting - // Some code - #endregion -} -")); - - LinkValidatorService service = new LinkValidatorService(_serviceProvider, _config, _fileService, _console); - - //Act - int line = 124; - int column = 3381; - Hyperlink link = new Hyperlink($"{_fileServiceMock.Root}\\index.md", line, column, "[!code-csharp[Code](scripts/foo.cs)]"); - await service.VerifyHyperlink(link); - - // Assert - service.Errors.Where(x => x.Severity == MarkdownErrorSeverity.Error).Should().BeEmpty(); - } - private ICustomConsoleLogger GetMockedConsoleLogger() { Mock console = new Mock(); From 93c851210801298977c3dd4aaaf33ec10e3df8c3 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Wed, 17 Jan 2024 16:07:34 +0100 Subject: [PATCH 3/3] Removed test for code links still being investigated --- .../DocLinkChecker.Test/MarkdownTests.cs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/DocLinkChecker/DocLinkChecker.Test/MarkdownTests.cs b/src/DocLinkChecker/DocLinkChecker.Test/MarkdownTests.cs index 13e9960..9c713d6 100644 --- a/src/DocLinkChecker/DocLinkChecker.Test/MarkdownTests.cs +++ b/src/DocLinkChecker/DocLinkChecker.Test/MarkdownTests.cs @@ -54,17 +54,6 @@ public class MarkdownTests .AddNewLine() .AddParagraphs(2); - private string _codeLinkingDocument = string.Empty - .AddHeading("Code Linking Document", 1) - .AddParagraphs(1).AddCodeLink("Code", "scripts/foo.cs") - .AddParagraphs(1).AddCodeLink("Code", "scripts/foo.cs?name=Bravo") - .AddNewLine() - .AddParagraphs(1).AddCodeLink(string.Empty, "scripts/bar.cs#starting") - .AddParagraphs(1).AddCodeLink(string.Empty, "scripts/bar.cs#L3-5") - .AddParagraphs(1).AddCodeLink(string.Empty, "scripts/bar.cs?highlight=2,5-7,9-)") - .AddNewLine() - .AddParagraphs(2); - [Fact] public void FindAllLinks() { @@ -131,12 +120,5 @@ public void FindAllTablesWithErrors() var result = MarkdownHelper.ParseMarkdownString(string.Empty, _errorDocument, true); result.errors.Count.Should().Be(5); } - - [Fact] - public void FindAllCodeLinks() - { - var result = MarkdownHelper.ParseMarkdownString(string.Empty, _codeLinkingDocument, false); - result.errors.Should().BeEmpty(); - } } } \ No newline at end of file