From 1ac689e25c150f7a018d88a1806cb4d8c9a71834 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Mon, 18 Nov 2024 14:42:51 +0100 Subject: [PATCH 01/12] WIP: initial setup of the DocAssembler tool --- src/DocAssembler/DocAssembler.sln | 25 ++ .../DocAssembler/Actions/ActionException.cs | 40 ++ .../DocAssembler/Actions/AssembleAction.cs | 63 ++++ .../DocAssembler/Actions/ConfigInitAction.cs | 59 +++ .../DocAssembler/DocAssembler.csproj | 42 +++ .../DocAssembler/FileService/FileService.cs | 78 ++++ .../DocAssembler/FileService/IFileService.cs | 70 ++++ .../DocAssembler/GlobalSuppressions.cs | 16 + src/DocAssembler/DocAssembler/Program.cs | 159 ++++++++ src/DocAssembler/DocAssembler/README.md | 355 ++++++++++++++++++ src/DocAssembler/DocAssembler/ReturnCode.cs | 22 ++ .../DocAssembler/Utils/LogUtil.cs | 44 +++ src/DocAssembler/DocAssembler/stylecop.json | 13 + 13 files changed, 986 insertions(+) create mode 100644 src/DocAssembler/DocAssembler.sln create mode 100644 src/DocAssembler/DocAssembler/Actions/ActionException.cs create mode 100644 src/DocAssembler/DocAssembler/Actions/AssembleAction.cs create mode 100644 src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs create mode 100644 src/DocAssembler/DocAssembler/DocAssembler.csproj create mode 100644 src/DocAssembler/DocAssembler/FileService/FileService.cs create mode 100644 src/DocAssembler/DocAssembler/FileService/IFileService.cs create mode 100644 src/DocAssembler/DocAssembler/GlobalSuppressions.cs create mode 100644 src/DocAssembler/DocAssembler/Program.cs create mode 100644 src/DocAssembler/DocAssembler/README.md create mode 100644 src/DocAssembler/DocAssembler/ReturnCode.cs create mode 100644 src/DocAssembler/DocAssembler/Utils/LogUtil.cs create mode 100644 src/DocAssembler/DocAssembler/stylecop.json diff --git a/src/DocAssembler/DocAssembler.sln b/src/DocAssembler/DocAssembler.sln new file mode 100644 index 0000000..6c56fb3 --- /dev/null +++ b/src/DocAssembler/DocAssembler.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.11.35431.28 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DocAssembler", "DocAssembler\DocAssembler.csproj", "{20348289-FB98-4EE3-987D-576E3C568EB3}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {20348289-FB98-4EE3-987D-576E3C568EB3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {20348289-FB98-4EE3-987D-576E3C568EB3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {20348289-FB98-4EE3-987D-576E3C568EB3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {20348289-FB98-4EE3-987D-576E3C568EB3}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {97487205-BC8C-4B1C-B40E-EDC19E4F01B9} + EndGlobalSection +EndGlobal diff --git a/src/DocAssembler/DocAssembler/Actions/ActionException.cs b/src/DocAssembler/DocAssembler/Actions/ActionException.cs new file mode 100644 index 0000000..a55bec9 --- /dev/null +++ b/src/DocAssembler/DocAssembler/Actions/ActionException.cs @@ -0,0 +1,40 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using System.Diagnostics.CodeAnalysis; + +namespace DocAssembler.Actions; + +/// +/// Exception class for the ParserService. +/// +[ExcludeFromCodeCoverage] +public class ActionException : Exception +{ + /// + /// Initializes a new instance of the class. + /// + public ActionException() + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Message of exception. + public ActionException(string message) + : base(message) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Message of exception. + /// Inner exception. + public ActionException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs new file mode 100644 index 0000000..fb78b7d --- /dev/null +++ b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs @@ -0,0 +1,63 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using DocAssembler.FileService; +using Microsoft.Extensions.Logging; + +namespace DocAssembler.Actions; + +/// +/// Assemble documentation in the output folder. The tool will also fix links following configuration. +/// +public class AssembleAction +{ + private readonly string _configFile; + private readonly string? _outFolder; + + private readonly IFileService? _fileService; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// Configuration file. + /// Output folder. + /// File service. + /// Logger. + public AssembleAction( + string configFile, + string? outFolder, + IFileService fileService, + ILogger logger) + { + _configFile = configFile; + _outFolder = outFolder; + + _fileService = fileService; + _logger = logger; + } + + /// + /// Run the action. + /// + /// 0 on success, 1 on warning, 2 on error. + public Task RunAsync() + { + ReturnCode ret = ReturnCode.Normal; + _logger.LogInformation($"\n*** INVENTORY STAGE."); + + try + { + ret = ReturnCode.Warning; + } + catch (Exception ex) + { + _logger.LogCritical($"Inventory error: {ex.Message}."); + ret = ReturnCode.Error; + } + + _logger.LogInformation($"END OF INVENTORY STAGE. Result: {ret}"); + return Task.FromResult(ret); + } +} diff --git a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs new file mode 100644 index 0000000..02cfca8 --- /dev/null +++ b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs @@ -0,0 +1,59 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using DocAssembler.FileService; +using Microsoft.Extensions.Logging; + +namespace DocAssembler.Actions; + +/// +/// Initialize and save an initial configuration file if it doesn't exist yet. +/// +public class ConfigInitAction +{ + private readonly string _outFolder; + + private readonly IFileService? _fileService; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// Output folder. + /// File service. + /// Logger. + public ConfigInitAction( + string outFolder, + IFileService fileService, + ILogger logger) + { + _outFolder = outFolder; + + _fileService = fileService; + _logger = logger; + } + + /// + /// Run the action. + /// + /// 0 on success, 1 on warning, 2 on error. + public Task RunAsync() + { + ReturnCode ret = ReturnCode.Normal; + _logger.LogInformation($"\n*** INVENTORY STAGE."); + + try + { + ret = ReturnCode.Warning; + } + catch (Exception ex) + { + _logger.LogCritical($"Inventory error: {ex.Message}."); + ret = ReturnCode.Error; + } + + _logger.LogInformation($"END OF INVENTORY STAGE. Result: {ret}"); + return Task.FromResult(ret); + } +} diff --git a/src/DocAssembler/DocAssembler/DocAssembler.csproj b/src/DocAssembler/DocAssembler/DocAssembler.csproj new file mode 100644 index 0000000..eb5989f --- /dev/null +++ b/src/DocAssembler/DocAssembler/DocAssembler.csproj @@ -0,0 +1,42 @@ + + + + Exe + net8.0 + 12.0 + true + true + enable + enable + true + latest-Recommended + + MIT + README.md + DocFx Companion Tools contributors + DocFx Companion Tools + DocAssembler + git + https://github.com/Ellerbach/docfx-companion-tools + https://github.com/Ellerbach/docfx-companion-tools + Tool to assemble documentation from various locations and change links where necessary. + docfx tools companion documentation assembler + + + + + + + + + + + + + + + + + + + diff --git a/src/DocAssembler/DocAssembler/FileService/FileService.cs b/src/DocAssembler/DocAssembler/FileService/FileService.cs new file mode 100644 index 0000000..ecce860 --- /dev/null +++ b/src/DocAssembler/DocAssembler/FileService/FileService.cs @@ -0,0 +1,78 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.FileSystemGlobbing; + +namespace DocAssembler.FileService; + +/// +/// File service implementation working with class. +/// +[ExcludeFromCodeCoverage] +public class FileService : IFileService +{ + /// + public string GetFullPath(string path) + { + return Path.GetFullPath(path); + } + + /// + public bool ExistsFileOrDirectory(string path) + { + return File.Exists(path) || Directory.Exists(path); + } + + /// + public IEnumerable GetFiles(string root, List includes, List excludes) + { + string fullRoot = Path.GetFullPath(root); + Matcher matcher = new(); + foreach (string folderName in includes) + { + matcher.AddInclude(folderName); + } + + foreach (string folderName in excludes) + { + matcher.AddExclude(folderName); + } + + // make sure we normalize the directory separator + return matcher.GetResultsInFullPath(fullRoot) + .Select(x => x.Replace("\\", "/")) + .ToList(); + } + + /// + public IEnumerable GetDirectories(string folder) + { + return Directory.GetDirectories(folder); + } + + /// > + public string ReadAllText(string path) + { + return File.ReadAllText(path); + } + + /// + public string[] ReadAllLines(string path) + { + return File.ReadAllLines(path); + } + + /// + public void WriteAllText(string path, string content) + { + File.WriteAllText(path, content); + } + + /// + public Stream OpenRead(string path) + { + return File.OpenRead(path); + } +} diff --git a/src/DocAssembler/DocAssembler/FileService/IFileService.cs b/src/DocAssembler/DocAssembler/FileService/IFileService.cs new file mode 100644 index 0000000..6e2b1be --- /dev/null +++ b/src/DocAssembler/DocAssembler/FileService/IFileService.cs @@ -0,0 +1,70 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +namespace DocAssembler.FileService; + +/// +/// File service interface. This is to hide file system access behind an interface. +/// This allows for the implementation of a mock for unit testing. +/// +public interface IFileService +{ + /// + /// Get the full path of the given path. + /// + /// Path of file or folder. + /// The full path of the file or folder. + string GetFullPath(string path); + + /// + /// Check if the given path exists as file or directory. + /// + /// Path to check. + /// A value indicating whether the path exists. + bool ExistsFileOrDirectory(string path); + + /// + /// Get files with the Glob File Pattern. + /// + /// Root path. + /// Include patterns. + /// Exclude patterns. + /// List of files. + IEnumerable GetFiles(string root, List includes, List excludes); + + /// + /// Get directories in the given path. + /// + /// Folder path. + /// List of folders. + IEnumerable GetDirectories(string folder); + + /// + /// Read the file as text string. + /// + /// Path of the file. + /// Contents of the file or empty if doesn't exist. + string ReadAllText(string path); + + /// + /// Read the file as array of strings split on newlines. + /// + /// Path of the file. + /// All lines of text or empty if doesn't exist. + string[] ReadAllLines(string path); + + /// + /// Write content to given path. + /// + /// Path of the file. + /// Content to write to the file. + void WriteAllText(string path, string content); + + /// + /// Get a stream for the given path to read. + /// + /// Path of the file. + /// A . + Stream OpenRead(string path); +} diff --git a/src/DocAssembler/DocAssembler/GlobalSuppressions.cs b/src/DocAssembler/DocAssembler/GlobalSuppressions.cs new file mode 100644 index 0000000..772da81 --- /dev/null +++ b/src/DocAssembler/DocAssembler/GlobalSuppressions.cs @@ -0,0 +1,16 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. +using System.Diagnostics.CodeAnalysis; + +[assembly: SuppressMessage("Performance", "CA1848:Use the LoggerMessage delegates", Justification = "No need to optimaze in console app.")] +[assembly: SuppressMessage("Usage", "CA2254:Template should be a static expression", Justification = "No need to optimize in console app.")] +[assembly: SuppressMessage("StyleCop.CSharp.NamingRules", "SA1309:Field names should not begin with underscore", Justification = "Coding style different")] +[assembly: SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1101:Prefix local calls with this", Justification = "We don't want this.")] +[assembly: SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "We will decide case by case.")] diff --git a/src/DocAssembler/DocAssembler/Program.cs b/src/DocAssembler/DocAssembler/Program.cs new file mode 100644 index 0000000..d200c4c --- /dev/null +++ b/src/DocAssembler/DocAssembler/Program.cs @@ -0,0 +1,159 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using System.CommandLine; +using System.CommandLine.Invocation; +using System.CommandLine.Parsing; +using DocAssembler; +using DocAssembler.Actions; +using DocAssembler.FileService; +using DocAssembler.Utils; +using Microsoft.Extensions.Logging; + +var logLevel = LogLevel.Warning; + +// parameters/options +var configFile = new Option( + name: "--config", + description: "The configuration file for the assembled documentation.") +{ + IsRequired = true, +}; +configFile.AddAlias("-c"); + +var outputFolder = new Option( + name: "--outfolder", + description: "Override the output folder for the assembled documentation in the config file."); +outputFolder.AddAlias("-o"); + +var verboseOption = new Option( + name: "--verbose", + description: "Show verbose messages of the process."); +verboseOption.AddAlias("-v"); + +// construct the root command +var rootCommand = new RootCommand( + """ + DocAssembler. + Assemble documentation in the output folder. The tool will also fix links following configuration. + + Return values: + 0 - succesfull. + 1 - some warnings, but process could be completed. + 2 - a fatal error occurred. + """); + +rootCommand.AddOption(configFile); +rootCommand.AddOption(outputFolder); +rootCommand.AddOption(verboseOption); + +var initCommand = new Command("init", "Intialize a configuration file in the current directory if it doesn't exist yet."); +rootCommand.Add(initCommand); + +// handle the execution of the root command +rootCommand.SetHandler(async (context) => +{ + // setup logging + SetLogLevel(context); + + LogParameters( + context.ParseResult.GetValueForOption(configFile)!, + context.ParseResult.GetValueForOption(outputFolder)); + + // execute the generator + context.ExitCode = (int)await AssembleDocumentationAsync( + context.ParseResult.GetValueForOption(configFile)!, + context.ParseResult.GetValueForOption(outputFolder)); +}); + +// handle the execution of the root command +initCommand.SetHandler(async (context) => +{ + // setup logging + SetLogLevel(context); + + // execute the configuration file initializer + context.ExitCode = (int)await GenerateConfigurationFile(); +}); + +return await rootCommand.InvokeAsync(args); + +// main process for configuration file generation. +async Task GenerateConfigurationFile() +{ + // setup services + ILogger logger = GetLogger(); + IFileService fileService = new FileService(); + + try + { + // the actual generation of the configuration file + ConfigInitAction action = new(string.Empty, fileService, logger); + ReturnCode ret = await action.RunAsync(); + + logger.LogInformation($"Command completed. Return value: {ret}."); + return ret; + } + catch (Exception ex) + { + logger.LogCritical(ex.Message); + return ReturnCode.Error; + } +} + +// main process for assembling documentation. +async Task AssembleDocumentationAsync( + FileInfo configFile, + DirectoryInfo? outputFolder) +{ + // setup services + ILogger logger = GetLogger(); + IFileService fileService = new FileService(); + + try + { + // WIP: should be inventory followed by assemble. + AssembleAction assemble = new(configFile.FullName, outputFolder?.FullName, fileService, logger); + ReturnCode ret = await assemble.RunAsync(); + + logger.LogInformation($"Command completed. Return value: {ret}."); + + return ret; + } + catch (Exception ex) + { + logger.LogCritical(ex.Message); + return ReturnCode.Error; + } +} + +// output logging of parameters +void LogParameters( + FileInfo configFile, + DirectoryInfo? outputFolder) +{ + ILogger logger = GetLogger(); + + logger!.LogInformation($"Configuration: {configFile.FullName}"); + if (outputFolder != null) + { + logger!.LogInformation($"Output folder: {outputFolder.FullName}"); + return; + } +} + +void SetLogLevel(InvocationContext context) +{ + if (context.ParseResult.GetValueForOption(verboseOption)) + { + logLevel = LogLevel.Debug; + } + else + { + logLevel = LogLevel.Warning; + } +} + +ILoggerFactory GetLoggerFactory() => LogUtil.GetLoggerFactory(logLevel); +ILogger GetLogger() => GetLoggerFactory().CreateLogger(nameof(DocAssembler)); diff --git a/src/DocAssembler/DocAssembler/README.md b/src/DocAssembler/DocAssembler/README.md new file mode 100644 index 0000000..dcdfeaf --- /dev/null +++ b/src/DocAssembler/DocAssembler/README.md @@ -0,0 +1,355 @@ +# Table of Contents (TOC) generator for DocFX + +This tool allow to generate a yaml compatible `toc.yml` file for DocFX. + +## Usage + +```text +DocAssembler [options] + +Options: + -d, --docfolder (REQUIRED) The root folder of the documentation. + -o, --outfolder The output folder for the generated table of contents + file. Default is the documentation folder. + -v, --verbose Show verbose messages of the process. + -s, --sequence Use .order files per folder to define the sequence of + files and directories. Format of the file is filename + without extension per line. + -r, --override Use .override files per folder to define title overrides + for files and folders. Format of the file is filename + without extension or directory name followed by a + semi-column followed by the custom title per line. + -g, --ignore Use .ignore files per folder to ignore directories. + Format of the file is directory name per line. + --indexing When to generated an index.md for a folder. + NoDefault - When no index.md or readme.md found. + NoDefaultMulti - When no index.md or readme.md found and + multiple files. + EmptyFolders - For empty folders. + NotExists - When no index found. + NotExistMulti - When no index and multiple files. + [default: Never] + --folderRef Strategy for folder-entry references. + None - Never reference anything. + Index - Index.md only if exists. + IndexReadme - Index.md or readme.md if exists. + First - First file in folder if any exists. + [default: First] + --ordering How to order items in a folder. + All - Folders and files combined. + FoldersFirst - Folders first, then files. + FilesFirst - Files first, then folders. [default: All] + -m, --multitoc Indicates how deep in the tree toc files should be + generated for those folders. A depth of 0 is the root + only (default behavior). + --camelCase Use camel casing for titles. + --version Show version information + -?, -h, --help Show help and usage information +``` + +Return values: + 0 - succesfull. + 1 - some warnings, but process could be completed. + 2 - a fatal error occurred. + +## Warnings, errors and verbose + +If the tool encounters situations that might need some action, a warning is written to the output. The table of contents is still created. If the tool encounters an error, an error message is written to the output. The table of contents will not be created. + +If you want to trace what the tool is doing, use the `-v or --verbose` flag to output all details of processing the files and folders and creating the table of contents. + +## Overall process + +The overall process of this tool is: + +1. Content inventory - retrieve all folders and files (`*.md` and `*swagger.json`) in the given documentation folder. Flags `-s | --sequence`, `-r | --override` and `-g | --ignore` are processed here to read setting files in the hierarchy. +2. Ensure indexing - validate structure with given settings. Depending on the `--indexing` flag automated `index.md` files are added where necessary. +3. Generate the table of contents - generate the `toc.yml` file(s). For folders it can be indicated if they should have a reference into child files using the `--folderRef` flag. Using the `--ordering` flag the ordering of directories and files can be defined. In this step the `-m | --multitoc ` flag is evaluated and processed on generation. + +### Title of directories and files + +For directories the name of the directory is used by default, where the first character is uppercased and special characters (`[`, `]`, `:`, \`,`\`, `{`, `}`, `(`, `)`, `*`, `/`) are removed and `-`, `_` and multiple spaces are replaced by a single space. + +For markdown files the first level-1 heading is taken as title. For swagger files the title and version are taken as title. On error the file name without extension is taken and processed the same way as the name of a directory. + +The `.override` setting file can be used to override this behavior. See [Defining title overrides with `.override`](#defining-title-overrides-with-override). + +## Folder settings + +Folder settings can be provided on ordering directories and files, ignore directories and override titles of files. Flags `-s | --sequence`, `-r | --override` and `-g | --ignore` are processed here to read setting files in the hierarchy. + +### Defining the order with `.order` + +If the `-s | --sequence` parameter is provided, the tool will inspect folders if a `.order` file exists and use that to determine the order of files and directories. The `.order` file is just a list of file- and/or directory-names, *case-sensitive* without file extensions. Also see the [Azure DevOps WIKI documentation on this file](https://docs.microsoft.com/en-us/azure/devops/project/wiki/wiki-file-structure?view=azure-devops#order-file). + +A sample `.order` file looks like this: + +```text +getting-started +working-agreements +developer +``` + +Ordering of directories and files in a folder is influenced by the `-s | --sequence` flag in combination with the `.order` file in that directory, combined with the (optional) `--ordering` flag. Also see [Ordering](#ordering). + +### Defining directories to ignore with `.ignore` + +If the `-g | --ignore` parameter is provided, the tool will inspect folders if a `.ignore` file exists and use that to ignore directories. The `.ignore` file is just a list of file- and/or directory-names, *case-sensitive* without file extensions. + +A sample `.ignore` file looks like this: + +```text +node_modules +bin +``` + +It only applies to the folder it's in, not for other subfolders under that folder. + +### Defining title overrides with `.override` + +If the `-r | --override` parameter is provided, the tool will inspect folders if a `.override` file exists and use that for overrides of file or directory titles as they will show in the generated `toc.yml`. The `.override` file is a list of file- and/or directory-names, *case-sensitive* without file extensions, followed by a semi-column, followed by the title to use. + +For example, if the folder name is `introduction`, the default behavior will be to create the name `Introduction`. If you want to call it `To start with`, you can use overrides, like in the following example: + +```text +introduction;To start with +working-agreements;All working agreements of all teams +``` + +The title for an MD-file is taken from the H1-header in the file. The title for a directory is the directory-name, but cleanup from special characters and the first character in capitals. + +## Automatic generating `index.md` files + +If the `-indexing ` parameter is provided the `method` defines the conditions for generating an `index.md` file. The options are: + +* `Never` - never generate an `index.md`. This is the default. +* `NoDefault` - generate an `index.md` when no `index.md` or `readme.md` is found in a folder. +* `NoDefaultMulti` - generate an `index.md` when no `index.md` or `readme.md` is found in a folder and there are 2 or more files. +* `NotExists` - generate an `index.md` when no `index.md` file is found in a folder. +* `NotExistsMulti` - generate an `index.md` when no `index.md` file is found in a folder and there are 2 or more files. +* `EmptyFolders` - generate an `index.md` when a folder doesn't contain any files. + +### Template for generating an `index.md` + +When an `index.md` file is generated, this is done by using a [Liquid template](https://shopify.github.io/liquid/). The tool contains a *default template*: + +```liquid +# {{ current.DisplayName }} + +{% comment -%}Looping through all the files and show the display name.{%- endcomment -%} +{% for file in current.Files -%} +{%- if file.IsMarkdown -%} +* [{{ file.DisplayName }}]({{ file.Name }}) +{% endif -%} +{%- endfor %} +``` + +This results in a markdown file like this: + +```markdown +# Brasil + +* [Nova Friburgo](nova-friburgo.md) +* [Rio de Janeiro](rio-de-janeiro.md) +* [Sao Paulo](sao-paulo.md) +``` + +You can also provide a customized template to be used. The ensure indexing process will look for a file with the name `.index.liquid` in the folder where an `index.md` needs to be generated. If it doesn't exist in that folder it's traversing all parent folders up to the root and until a `.index.liquid` file is found. + +In the template access is provided to this information: + +* `current` - this is the current folder that needs an `index.md` file of type `FolderData`. +* `root` - this is the root folder of the complete hierarchy of the documentation of type `FolderData`. + +#### `FolderData` class + +| Property | Description | +| -------------- | ------------------------------------------------------------ | +| `Name` | Folder name from disk | +| `DisplayName` | Title of the folder | +| `Path` | Full path of the folder | +| `Sequence` | Sequence number from the `.order` file or `int.MaxValue` when not defined. | +| `RelativePath` | Relative path of the folder from the root of the documentation. | +| `Parent` | Parent folder. When `null` it's the root folder. | +| `Folders` | A list of `FolderData` objects for the sub-folders in this folder. | +| `Files` | A list of `FileData` objects for the files in this folder. | +| `HasIndex` | A `boolean` indicating whether this folder contains an `index.md` | +| `Index` | The `FileData` object of the `index.md` in this folder if it exists. If it doesn't exists this will be `null`. | +| `HasReadme` | A `boolean` indicating whether this folder contains an `README.md` | +| `Readme` | The `FileData` object of the `README.md` in this folder if it exists. If it doesn't exists this will be `null`. | + +#### `FileData` class + +| Property | Description | +| -------------- | ------------------------------------------------------------ | +| `Name` | Filename including the extension | +| `DisplayName` | Title of the file. | +| `Path` | Full path of the file | +| `Sequence` | Sequence number from the `.order` file or `int.MaxValue` when not defined. | +| `RelativePath` | Relative path of the file from the root of the documentation. | +| `Parent` | Parent folder. | +| `IsMarkdown` | A `boolean` indicating whether this file is a markdown file. | +| `IsSwagger` | A `boolean` indicating whether this file is a Swagger JSON file. | +| `IsIndex` | A `boolean` indicating whether this file is an `index.md` file. | +| `IsReadme` | A `boolean` indicating whether this file is a `README.md` file. | + +For more information on how to use Liquid logic, see the article [Using Liquid for text-based templates with .NET | by Martin Tirion | Medium](https://mtirion.medium.com/using-liquid-for-text-base-templates-with-net-80ae503fa635) and the [Liquid reference](https://shopify.github.io/liquid/basics/introduction/). + +Liquid, by design, is very forgiving. If you reference an object or property that doesn't exist, it will render to an empty string. But if you introduce language errors (missing `{{` for instance) an error is thrown, the error is in the output of the tool but will not crash the tool, but will be resulting in error code 1 (warning). In the case of an error like this, no `index.md` is generated. + +## Ordering + +There are these options for ordering directories and folders: + +* `All` - order all directories and files by sequence, then by title. +* `FoldersFirst` - order all directories first, then the files. Ordering is for each of them done by sequence, then by title. +* `FilesFirst` - order all files first, then the folders. Ordering is for each of them done by sequence, then by title. + +For all of these options the `.order` file can be used when it exists and the `-s | --sequence` flag is used. The line in the `.order` file determines the sequence of a file or directory. So, the first entry results in sequence 1. In all other cases a folder or file has an equal sequence of `int.MaxValue`. + +By default the ordering of files is applied where the `index.md` is first and the `README.md` is second, optionally followed by the settings from the `.order` file. This behavior can only be overruled by adding `index` and/or `readme` to a `.order` file and use of the `-s | --sequence` flag. + +> [!NOTE] +> +> `README` and `index` are always validated **case-sensitive** to make sure they are ordered correctly. All other file names and directory names are matched **case-insensitive**. + +## Folder referencing + +The table of content is constructed from the folders and files. For folders there are various strategies to determine if it will have a reference: + +* `None` - no reference for all folders. +* `Index` - reference the `index.md` in the folder if it exists. +* `IndexReadme` - reference the `index.md` if it exists, otherwise reference the `README.md` if it exists. +* `First` - reference the first file in the folder after [ordering](#ordering) has been applied. + +When using DocFx to generate the website, folders with no reference will just be entries in the hive that can be opened and closed. The UI will determine what will be showed as content. + +## Multiple table of content files + +The default for this tool is to generate only one `toc.yml` file in the root of the output directory. But with a large hierarchy, this file can get pretty large. In that case it might be easier to have a few `toc.yml` files per level to have multiple, smaller `toc.yml` files. + +The `-m | --multitoc` option will control how far down the hierarchy `toc.yml` files are generated. Let's explain this feature by an example hierarchy: + +```text +📂docs + 📄README.md + 📂continents + 📄index.md + 📂americas + 📄README.md + 📄extra-facts.md + 📂brasil + 📄README.md + 📄nova-friburgo.md + 📄rio-de-janeiro.md + 📂united-states + 📄los-angeles.md + 📄new-york.md + 📄washington.md + 📂europe + 📄README.md + 📂germany + 📄berlin.md + 📄munich.md + 📂netherlands + 📄amsterdam.md + 📄rotterdam.md + 📂vehicles + 📄index.md + 📂cars + 📄README.md + 📄audi.md + 📄bmw.md +``` + +### Default behavior or depth=0 + +By default, when the `depth` is `0` (or the option is omitted), only one `toc.yml` file is generated in the root of the output folder containing the complete hierarchy of folders and files. For the example hierarchy it would look like this: + +```yaml +# This is an automatically generated file +- name: Multi toc example + href: README.md +- name: Continents + href: continents/index.md + items: + - name: Americas + href: continents/americas/README.md + items: + - name: Americas Extra Facts + href: continents/americas/extra-facts.md + - name: Brasil + href: continents/americas/brasil/README.md + items: + - name: Nova Friburgo + href: continents/americas/brasil/nova-friburgo.md + - name: Rio de Janeiro + href: continents/americas/brasil/rio-de-janeiro.md + - name: Los Angeles + href: continents/americas/united-states/los-angeles.md + items: + - name: New York + href: continents/americas/united-states/new-york.md + - name: Washington + href: continents/americas/united-states/washington.md + - name: Europe + href: continents/europe/README.md + items: + - name: Amsterdam + href: continents/europe/netherlands/amsterdam.md + items: + - name: Rotterdam + href: continents/europe/netherlands/rotterdam.md + - name: Berlin + href: continents/europe/germany/berlin.md + items: + - name: Munich + href: continents/europe/germany/munich.md +- name: Vehicles + href: vehicles/index.md + items: + - name: Cars + href: vehicles/cars/README.md + items: + - name: Audi + href: vehicles/cars/audi.md + - name: BMW + href: vehicles/cars/bmw.md + +``` + +### Behavior with depth=1 or more + +When a `depth` of `1` is given, a `toc.yml` is generated in the root of the output folder and in each sub-folder of the documentation root. The `toc.yml` in the root will only contain documents of the folder itself and references to the `toc.yml` files in the sub-folders. In our example for the root it would look like this: + +```yaml +# This is an automatically generated file +- name: Multi toc example + href: README.md +- name: Continents + href: continents/toc.yml +- name: Vehicles + href: vehicles/toc.yml +``` + +The `toc.yml` files in the sub-folders `continents` and `vehicles` will contain the complete hierarchy from that point on. For instance, for `vehicles` it will look like this: + +```yaml +# This is an automatically generated file +- name: Cars + href: cars/README.md + items: + - name: Audi + href: cars/audi.md + - name: BMW + href: cars/bmw.md +``` + +## Camel case titles + +By default titles are changed to pascal casing, meaning that the first character is capitalized. With the option `--camelCase` all titles will be changed to camel casing, meaning that the first character is lower cased. Only exception are overrides from `.override` files. + +> [!NOTE] +> +> As this rule is applied to everything, it is also applied to titles coming from Swagger-files. If this is an issue, this can be corrected for that file using an `.override` file in that folder. diff --git a/src/DocAssembler/DocAssembler/ReturnCode.cs b/src/DocAssembler/DocAssembler/ReturnCode.cs new file mode 100644 index 0000000..3a72073 --- /dev/null +++ b/src/DocAssembler/DocAssembler/ReturnCode.cs @@ -0,0 +1,22 @@ +namespace DocAssembler; + +/// +/// Return code for the application. +/// +public enum ReturnCode +{ + /// + /// All went well. + /// + Normal = 0, + + /// + /// A few warnings, but process completed. + /// + Warning = 1, + + /// + /// An error occurred, process not completed. + /// + Error = 2, +} diff --git a/src/DocAssembler/DocAssembler/Utils/LogUtil.cs b/src/DocAssembler/DocAssembler/Utils/LogUtil.cs new file mode 100644 index 0000000..a39cf74 --- /dev/null +++ b/src/DocAssembler/DocAssembler/Utils/LogUtil.cs @@ -0,0 +1,44 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using Microsoft.Extensions.Logging; +using Serilog; +using Serilog.Events; + +namespace DocAssembler.Utils; + +/// +/// Log utils. +/// +[ExcludeFromCodeCoverage] +internal static class LogUtil +{ + /// + /// Get the logger factory. + /// + /// Log level. + /// Logger factory. + /// When an unknown log level is given. + public static ILoggerFactory GetLoggerFactory(LogLevel logLevel1) + { + var serilogLevel = logLevel1 switch + { + LogLevel.Critical => LogEventLevel.Fatal, + LogLevel.Error => LogEventLevel.Error, + LogLevel.Warning => LogEventLevel.Warning, + LogLevel.Information => LogEventLevel.Information, + LogLevel.Debug => LogEventLevel.Debug, + LogLevel.Trace => LogEventLevel.Verbose, + _ => throw new ArgumentOutOfRangeException(nameof(logLevel1)), + }; + + var serilog = new LoggerConfiguration() + .MinimumLevel.Is(serilogLevel) + .WriteTo.Console(standardErrorFromLevel: LogEventLevel.Warning, outputTemplate: "{Message:lj}{NewLine}", formatProvider: CultureInfo.InvariantCulture) + .CreateLogger(); + return LoggerFactory.Create(p => p.AddSerilog(serilog)); + } +} diff --git a/src/DocAssembler/DocAssembler/stylecop.json b/src/DocAssembler/DocAssembler/stylecop.json new file mode 100644 index 0000000..3feaecc --- /dev/null +++ b/src/DocAssembler/DocAssembler/stylecop.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", + "settings": { + "documentationRules": { + "companyName": "DocFx Companion Tools", + "copyrightText": "Copyright (c) {companyName}. All rights reserved.\nLicensed under the {licenseName} license. See {licenseFile} file in the project root for full license information.", + "variables": { + "licenseName": "MIT", + "licenseFile": "LICENSE" + } + } + } +} From 114cf81b7e6418f148e94af9e4989b7db74f5219 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Wed, 20 Nov 2024 23:20:49 +0100 Subject: [PATCH 02/12] WIP: working on docassembler --- .../DocAssembler/Actions/ConfigInitAction.cs | 39 ++++++-- .../DocAssembler/Actions/InventoryAction.cs | 89 +++++++++++++++++++ .../Configuration/AssembleConfiguration.cs | 24 +++++ .../DocAssembler/Configuration/Content.cs | 54 +++++++++++ .../Configuration/FolderNamingStrategy.cs | 21 +++++ .../DocAssembler/FileService/FileData.cs | 21 +++++ .../FileService/FileInfoService.cs | 72 +++++++++++++++ .../DocAssembler/FileService/MarkdownLink.cs | 16 ++++ src/DocAssembler/DocAssembler/Program.cs | 52 +++++++---- .../DocAssembler/Utils/SerializationUtil.cs | 36 ++++++++ 10 files changed, 401 insertions(+), 23 deletions(-) create mode 100644 src/DocAssembler/DocAssembler/Actions/InventoryAction.cs create mode 100644 src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs create mode 100644 src/DocAssembler/DocAssembler/Configuration/Content.cs create mode 100644 src/DocAssembler/DocAssembler/Configuration/FolderNamingStrategy.cs create mode 100644 src/DocAssembler/DocAssembler/FileService/FileData.cs create mode 100644 src/DocAssembler/DocAssembler/FileService/FileInfoService.cs create mode 100644 src/DocAssembler/DocAssembler/FileService/MarkdownLink.cs create mode 100644 src/DocAssembler/DocAssembler/Utils/SerializationUtil.cs diff --git a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs index 02cfca8..10bd967 100644 --- a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs @@ -2,7 +2,9 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using DocAssembler.Configuration; using DocAssembler.FileService; +using DocAssembler.Utils; using Microsoft.Extensions.Logging; namespace DocAssembler.Actions; @@ -12,6 +14,8 @@ namespace DocAssembler.Actions; /// public class ConfigInitAction { + private const string CONFIGFILENAME = ".docassembler.json"; + private readonly string _outFolder; private readonly IFileService? _fileService; @@ -38,22 +42,45 @@ public ConfigInitAction( /// Run the action. /// /// 0 on success, 1 on warning, 2 on error. - public Task RunAsync() + public async Task RunAsync() { ReturnCode ret = ReturnCode.Normal; - _logger.LogInformation($"\n*** INVENTORY STAGE."); try { - ret = ReturnCode.Warning; + string path = Path.Combine(_outFolder, CONFIGFILENAME); + if (File.Exists(path)) + { + _logger.LogError($"*** ERROR: '{path}' already exists. We don't overwrite."); + + // indicate we're done with an error + return ReturnCode.Error; + } + + var config = new AssembleConfiguration + { + OutputFolder = "out", + Content = + [ + new Content + { + SourceFolder = "docs", + Files = { "**" }, + Naming = FolderNamingStrategy.ParentFolder, + ExternalFilePrefix = "https://github.com/example/blob/main/", + }, + ], + }; + + await File.WriteAllTextAsync(path, SerializationUtil.Serialize(config)); + _logger.LogInformation($"Initial configuration saved in '{path}'"); } catch (Exception ex) { - _logger.LogCritical($"Inventory error: {ex.Message}."); + _logger.LogCritical($"Saving initial configuration error: {ex.Message}."); ret = ReturnCode.Error; } - _logger.LogInformation($"END OF INVENTORY STAGE. Result: {ret}"); - return Task.FromResult(ret); + return ret; } } diff --git a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs new file mode 100644 index 0000000..3365885 --- /dev/null +++ b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs @@ -0,0 +1,89 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using DocAssembler.Configuration; +using DocAssembler.FileService; +using DocAssembler.Utils; +using Microsoft.Extensions.Logging; + +namespace DocAssembler.Actions; + +/// +/// Inventory action to retrieve configured content. +/// +public class InventoryAction +{ + private readonly string _workingFolder; + private readonly string _configFile; + private readonly string? _outputFolder; + private readonly FileInfoService _fileInfoService; + private readonly IFileService _fileService; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// Working folder. + /// Configuration file path. + /// Output folder override. + /// File service. + /// Logger. + public InventoryAction(string workingFolder, string configFile, string? outputFolder, IFileService fileService, ILogger logger) + { + _workingFolder = workingFolder; + _configFile = configFile; + _outputFolder = outputFolder; + _fileService = fileService; + _logger = logger; + + _fileInfoService = new(workingFolder, _fileService, _logger); + } + + /// + /// Run the action. + /// + /// 0 on success, 1 on warning, 2 on error. + public Task RunAsync() + { + ReturnCode ret = ReturnCode.Normal; + + try + { + AssembleConfiguration config = ReadConfigurationAsync(_configFile); + if (!string.IsNullOrWhiteSpace(_outputFolder)) + { + // overwrite output folder with given value + config.OutputFolder = _outputFolder; + } + + foreach (var content in config.Content) + { + var source = Path.Combine(_workingFolder, content.SourceFolder); + var files = _fileService.GetFiles(source, content.Files, content.Exclude); + foreach (var file in files) + { + _fileInfoService.GetExternalHyperlinks(source, file); + } + } + } + catch (Exception ex) + { + _logger.LogCritical($"Reading configuration error: {ex.Message}"); + ret = ReturnCode.Error; + } + + return Task.FromResult(ret); + } + + private AssembleConfiguration ReadConfigurationAsync(string configFile) + { + if (!_fileService.ExistsFileOrDirectory(configFile)) + { + throw new ActionException($"Configuration file '{configFile}' doesn't exist."); + } + + string json = _fileService.ReadAllText(configFile); + return SerializationUtil.Deserialize(json); + } +} diff --git a/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs b/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs new file mode 100644 index 0000000..b088f81 --- /dev/null +++ b/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs @@ -0,0 +1,24 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using System.Text.Json.Serialization; + +namespace DocAssembler.Configuration; + +/// +/// Assemble configuration. +/// +public sealed record AssembleConfiguration +{ + /// + /// Gets or sets the output folder. + /// + [JsonPropertyName("path")] + public string OutputFolder { get; set; } = string.Empty; + + /// + /// Gets or sets the content to process. + /// + public List Content { get; set; } = new(); +} diff --git a/src/DocAssembler/DocAssembler/Configuration/Content.cs b/src/DocAssembler/DocAssembler/Configuration/Content.cs new file mode 100644 index 0000000..403e9c7 --- /dev/null +++ b/src/DocAssembler/DocAssembler/Configuration/Content.cs @@ -0,0 +1,54 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using System.Text.Json.Serialization; + +namespace DocAssembler.Configuration; + +/// +/// Content definition using globbing patterns. +/// +public sealed record Content +{ + /// + /// Gets or sets the source folder. + /// + [JsonPropertyName("src")] + public string SourceFolder { get; set; } = string.Empty; + + /// + /// Gets or sets the optional destination folder. + /// + [JsonPropertyName("path")] + public string? DestinationFolder { get; set; } + + /// + /// Gets or sets the folders and files to include. + /// This list supports the file glob pattern. + /// + public List Files { get; set; } = new(); + + /// + /// Gets or sets the folders and files to exclude. + /// This list supports the file glob pattern. + /// + public List Exclude { get; set; } = new(); + + /// + /// Gets or sets naming strategy of the target folder. + /// + public FolderNamingStrategy Naming { get; set; } + + /// + /// Gets or sets a value indicating whether we need to fix markdown links. + /// + public bool FixMarkdownLinks { get; set; } = true; + + /// + /// Gets or sets the prefix for external files like source files. + /// This is for all references to files that are not part of the + /// documentation hierarchy. Use this to prefix Gitlab URL's. + /// + public string ExternalFilePrefix { get; set; } = string.Empty; +} diff --git a/src/DocAssembler/DocAssembler/Configuration/FolderNamingStrategy.cs b/src/DocAssembler/DocAssembler/Configuration/FolderNamingStrategy.cs new file mode 100644 index 0000000..0468b53 --- /dev/null +++ b/src/DocAssembler/DocAssembler/Configuration/FolderNamingStrategy.cs @@ -0,0 +1,21 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +namespace DocAssembler.Configuration; + +/// +/// Copy strategy. +/// +public enum FolderNamingStrategy +{ + /// + /// Use name of the source folder. + /// + SourceFolder, + + /// + /// Use name of the parent folder of the source. + /// + ParentFolder, +} diff --git a/src/DocAssembler/DocAssembler/FileService/FileData.cs b/src/DocAssembler/DocAssembler/FileService/FileData.cs new file mode 100644 index 0000000..f9063cf --- /dev/null +++ b/src/DocAssembler/DocAssembler/FileService/FileData.cs @@ -0,0 +1,21 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +namespace DocAssembler.FileService; + +/// +/// File data. +/// +public sealed record FileData +{ + /// + /// Gets or sets the original path of the file. + /// + public string OriginalPath { get; set; } = string.Empty; + + /// + /// Gets or sets the output path of the file. + /// + public string OutputPath { get; set; } = string.Empty; +} diff --git a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs new file mode 100644 index 0000000..16a6b9e --- /dev/null +++ b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs @@ -0,0 +1,72 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using Markdig; +using Markdig.Syntax; +using Markdig.Syntax.Inlines; +using Microsoft.Extensions.Logging; + +namespace DocAssembler.FileService; + +/// +/// File info service. +/// +public class FileInfoService +{ + private readonly string _workingFolder; + private readonly IFileService _fileService; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// Working folder. + /// File service. + /// Logger. + public FileInfoService(string workingFolder, IFileService fileService, ILogger logger) + { + _workingFolder = workingFolder; + _fileService = fileService; + _logger = logger; + } + + /// + /// Get the H1 title from a markdown file. + /// + /// Root path of the documentation. + /// File path of the markdown file. + /// First H1, or the filename as title if that fails. + public string GetExternalHyperlinks(string root, string filePath) + { + string markdownFilePath = _fileService.GetFullPath(filePath); + string markdown = _fileService.ReadAllText(markdownFilePath); + + MarkdownPipeline pipeline = new MarkdownPipelineBuilder() + .UseAdvancedExtensions() + .Build(); + MarkdownDocument document = Markdown.Parse(markdown, pipeline); + + // get all links + var links = document + .Descendants() + .Where(x => !x.UrlHasPointyBrackets && + x.Url != null && + !x.Url.StartsWith("https://", StringComparison.InvariantCulture) && + !x.Url.StartsWith("http://", StringComparison.InvariantCulture) && + !x.Url.StartsWith("ftps://", StringComparison.InvariantCulture) && + !x.Url.StartsWith("ftp://", StringComparison.InvariantCulture) && + !x.Url.StartsWith("mailto:", StringComparison.InvariantCulture) && + !x.Url.StartsWith("xref:", StringComparison.InvariantCulture)) + .ToList(); + + foreach (var link in links) + { + string url = _fileService.GetFullPath(Path.Combine(_workingFolder, link.Url!)); + Console.WriteLine($"{markdown.Substring(link.Span.Start, link.Span.Length)} => {url}"); + } + + // in case we couldn't get an H1 from markdown, return the filepath sanitized. + return $"Hello markdown!"; + } +} diff --git a/src/DocAssembler/DocAssembler/FileService/MarkdownLink.cs b/src/DocAssembler/DocAssembler/FileService/MarkdownLink.cs new file mode 100644 index 0000000..8b4267a --- /dev/null +++ b/src/DocAssembler/DocAssembler/FileService/MarkdownLink.cs @@ -0,0 +1,16 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +namespace DocAssembler.FileService; + +/// +/// Markdown link in document. +/// +public sealed record MarkdownLink +{ + /// + /// Gets or sets the URL. + /// + public string Url { get; set; } = string.Empty; +} diff --git a/src/DocAssembler/DocAssembler/Program.cs b/src/DocAssembler/DocAssembler/Program.cs index d200c4c..6fb9e1d 100644 --- a/src/DocAssembler/DocAssembler/Program.cs +++ b/src/DocAssembler/DocAssembler/Program.cs @@ -14,18 +14,20 @@ var logLevel = LogLevel.Warning; // parameters/options -var configFile = new Option( +var configFileOption = new Option( name: "--config", description: "The configuration file for the assembled documentation.") { IsRequired = true, }; -configFile.AddAlias("-c"); -var outputFolder = new Option( +var workingFolderOption = new Option( + name: "--workingfolder", + description: "The working folder. Default is the current folder."); + +var outputFolderOption = new Option( name: "--outfolder", description: "Override the output folder for the assembled documentation in the config file."); -outputFolder.AddAlias("-o"); var verboseOption = new Option( name: "--verbose", @@ -44,8 +46,9 @@ Assemble documentation in the output folder. The tool will also fix links follow 2 - a fatal error occurred. """); -rootCommand.AddOption(configFile); -rootCommand.AddOption(outputFolder); +rootCommand.AddOption(workingFolderOption); +rootCommand.AddOption(configFileOption); +rootCommand.AddOption(outputFolderOption); rootCommand.AddOption(verboseOption); var initCommand = new Command("init", "Intialize a configuration file in the current directory if it doesn't exist yet."); @@ -58,13 +61,15 @@ Assemble documentation in the output folder. The tool will also fix links follow SetLogLevel(context); LogParameters( - context.ParseResult.GetValueForOption(configFile)!, - context.ParseResult.GetValueForOption(outputFolder)); + context.ParseResult.GetValueForOption(configFileOption)!, + context.ParseResult.GetValueForOption(outputFolderOption), + context.ParseResult.GetValueForOption(workingFolderOption)); // execute the generator context.ExitCode = (int)await AssembleDocumentationAsync( - context.ParseResult.GetValueForOption(configFile)!, - context.ParseResult.GetValueForOption(outputFolder)); + context.ParseResult.GetValueForOption(configFileOption)!, + context.ParseResult.GetValueForOption(outputFolderOption), + context.ParseResult.GetValueForOption(workingFolderOption)); }); // handle the execution of the root command @@ -89,7 +94,7 @@ async Task GenerateConfigurationFile() try { // the actual generation of the configuration file - ConfigInitAction action = new(string.Empty, fileService, logger); + ConfigInitAction action = new(Environment.CurrentDirectory, fileService, logger); ReturnCode ret = await action.RunAsync(); logger.LogInformation($"Command completed. Return value: {ret}."); @@ -105,7 +110,8 @@ async Task GenerateConfigurationFile() // main process for assembling documentation. async Task AssembleDocumentationAsync( FileInfo configFile, - DirectoryInfo? outputFolder) + DirectoryInfo? outputFolder, + DirectoryInfo? workingFolder) { // setup services ILogger logger = GetLogger(); @@ -113,9 +119,14 @@ async Task AssembleDocumentationAsync( try { - // WIP: should be inventory followed by assemble. + ReturnCode ret = ReturnCode.Normal; + + string folder = workingFolder?.FullName ?? Directory.GetCurrentDirectory(); + InventoryAction inventory = new(folder, configFile.FullName, outputFolder?.FullName, fileService, logger); + ret &= await inventory.RunAsync(); + AssembleAction assemble = new(configFile.FullName, outputFolder?.FullName, fileService, logger); - ReturnCode ret = await assemble.RunAsync(); + ret &= await assemble.RunAsync(); logger.LogInformation($"Command completed. Return value: {ret}."); @@ -131,14 +142,21 @@ async Task AssembleDocumentationAsync( // output logging of parameters void LogParameters( FileInfo configFile, - DirectoryInfo? outputFolder) + DirectoryInfo? outputFolder, + DirectoryInfo? workingFolder) { ILogger logger = GetLogger(); - logger!.LogInformation($"Configuration: {configFile.FullName}"); + logger!.LogInformation($"Configuration : {configFile.FullName}"); if (outputFolder != null) { - logger!.LogInformation($"Output folder: {outputFolder.FullName}"); + logger!.LogInformation($"Output folder: {outputFolder.FullName}"); + return; + } + + if (workingFolder != null) + { + logger!.LogInformation($"Working folder: {workingFolder.FullName}"); return; } } diff --git a/src/DocAssembler/DocAssembler/Utils/SerializationUtil.cs b/src/DocAssembler/DocAssembler/Utils/SerializationUtil.cs new file mode 100644 index 0000000..889249f --- /dev/null +++ b/src/DocAssembler/DocAssembler/Utils/SerializationUtil.cs @@ -0,0 +1,36 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DocAssembler.Utils; + +/// +/// Serialization utilities. +/// +internal static class SerializationUtil +{ + /// + /// Gets the JSON serializer options. + /// + public static JsonSerializerOptions Options => new() + { + ReadCommentHandling = JsonCommentHandling.Skip, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true, + Converters = + { + new JsonStringEnumConverter(), + }, + }; + + /// + /// Serialize object. + /// + public static string Serialize(object value) => JsonSerializer.Serialize(value, Options); + + /// + /// Deserialize JSON string. + /// + /// Target type. + public static T Deserialize(string json) => JsonSerializer.Deserialize(json, Options)!; +} From 657a6f23aa5781ba6844874b5236b759dcaeb094 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Mon, 25 Nov 2024 18:11:34 +0100 Subject: [PATCH 03/12] WIP: first implementation working. Need to add tests --- .../DocAssembler/Actions/AssembleAction.cs | 62 +++- .../DocAssembler/Actions/ConfigInitAction.cs | 18 +- .../DocAssembler/Actions/InventoryAction.cs | 170 +++++++++- .../Configuration/AssembleConfiguration.cs | 6 +- .../DocAssembler/Configuration/Content.cs | 24 +- .../DocAssembler/FileService/FileData.cs | 20 +- .../FileService/FileInfoService.cs | 21 +- .../FileService/FilePathExtensions.cs | 36 +++ .../DocAssembler/FileService/FileService.cs | 20 +- .../DocAssembler/FileService/Hyperlink.cs | 298 ++++++++++++++++++ .../DocAssembler/FileService/HyperlinkType.cs | 47 +++ .../DocAssembler/FileService/IFileService.cs | 9 +- src/DocAssembler/DocAssembler/Program.cs | 2 +- 13 files changed, 669 insertions(+), 64 deletions(-) create mode 100644 src/DocAssembler/DocAssembler/FileService/FilePathExtensions.cs create mode 100644 src/DocAssembler/DocAssembler/FileService/Hyperlink.cs create mode 100644 src/DocAssembler/DocAssembler/FileService/HyperlinkType.cs diff --git a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs index fb78b7d..691a395 100644 --- a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs @@ -2,6 +2,8 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using System.Diagnostics; +using System.Text; using DocAssembler.FileService; using Microsoft.Extensions.Logging; @@ -12,28 +14,22 @@ namespace DocAssembler.Actions; /// public class AssembleAction { - private readonly string _configFile; - private readonly string? _outFolder; - - private readonly IFileService? _fileService; + private readonly List _files; + private readonly IFileService _fileService; private readonly ILogger _logger; /// /// Initializes a new instance of the class. /// - /// Configuration file. - /// Output folder. + /// List of files to process. /// File service. /// Logger. public AssembleAction( - string configFile, - string? outFolder, + List files, IFileService fileService, ILogger logger) { - _configFile = configFile; - _outFolder = outFolder; - + _files = files; _fileService = fileService; _logger = logger; } @@ -45,19 +41,55 @@ public AssembleAction( public Task RunAsync() { ReturnCode ret = ReturnCode.Normal; - _logger.LogInformation($"\n*** INVENTORY STAGE."); + _logger.LogInformation($"\n*** ASSEMBLE STAGE."); try { - ret = ReturnCode.Warning; + foreach (var file in _files) + { + // get all links that need to be changed + var updates = file.Links + .Where(x => !x.OriginalUrl.Equals(x.DestinationRelativeUrl ?? x.DestinationFullUrl, StringComparison.Ordinal)) + .OrderBy(x => x.UrlSpanStart); + if (updates.Any()) + { + var markdown = _fileService.ReadAllText(file.SourcePath); + StringBuilder sb = new StringBuilder(); + int pos = 0; + foreach (var update in updates) + { + // first append text so far from markdown + Console.WriteLine($"pos={pos} len={update.UrlSpanStart - pos} md={markdown.Length}"); + sb.Append(markdown.AsSpan(pos, update.UrlSpanStart - pos)); + + // append new link + sb.Append(update.DestinationRelativeUrl ?? update.DestinationFullUrl); + + // set new starting position + pos = update.UrlSpanEnd + 1; + } + + // add final part of markdown + sb.Append(markdown.AsSpan(pos)); + + Directory.CreateDirectory(Path.GetDirectoryName(file.DestinationPath)!); + _fileService.WriteAllText(file.DestinationPath, sb.ToString()); + _logger.LogInformation($"Copied '{file.SourcePath}' to '{file.DestinationPath}'. Replace {updates.Count()} links."); + } + else + { + _fileService.Copy(file.SourcePath, file.DestinationPath); + _logger.LogInformation($"Copied '{file.SourcePath}' to '{file.DestinationPath}'."); + } + } } catch (Exception ex) { - _logger.LogCritical($"Inventory error: {ex.Message}."); + _logger.LogCritical($"Assembly error: {ex.Message}."); ret = ReturnCode.Error; } - _logger.LogInformation($"END OF INVENTORY STAGE. Result: {ret}"); + _logger.LogInformation($"END OF ASSEMBLE STAGE. Result: {ret}"); return Task.FromResult(ret); } } diff --git a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs index 10bd967..fed5e34 100644 --- a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs @@ -59,14 +59,28 @@ public async Task RunAsync() var config = new AssembleConfiguration { - OutputFolder = "out", + DestinationFolder = "out", Content = [ + new Content + { + SourceFolder = ".docfx", + Files = { "**" }, + RawCopy = true, + }, new Content { SourceFolder = "docs", Files = { "**" }, - Naming = FolderNamingStrategy.ParentFolder, + ExternalFilePrefix = "https://github.com/example/blob/main/", + }, + new Content + { + SourceFolder = "backend", + DestinationFolder = "services", + Files = { "**/docs/**" }, + ReplacePattern = "/[Dd]ocs/", + ReplaceValue = "/", ExternalFilePrefix = "https://github.com/example/blob/main/", }, ], diff --git a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs index 3365885..94ff343 100644 --- a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs @@ -2,6 +2,7 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using System.Text.RegularExpressions; using DocAssembler.Configuration; using DocAssembler.FileService; using DocAssembler.Utils; @@ -21,25 +22,41 @@ public class InventoryAction private readonly IFileService _fileService; private readonly ILogger _logger; + private readonly AssembleConfiguration _config; + /// /// Initializes a new instance of the class. /// /// Working folder. /// Configuration file path. - /// Output folder override. + /// Output folder override. /// File service. /// Logger. - public InventoryAction(string workingFolder, string configFile, string? outputFolder, IFileService fileService, ILogger logger) + public InventoryAction(string workingFolder, string configFile, string? outputFolderOverride, IFileService fileService, ILogger logger) { _workingFolder = workingFolder; _configFile = configFile; - _outputFolder = outputFolder; _fileService = fileService; _logger = logger; _fileInfoService = new(workingFolder, _fileService, _logger); + + _config = ReadConfigurationAsync(_configFile); + if (!string.IsNullOrWhiteSpace(outputFolderOverride)) + { + // overwrite output folder with given override value + _config.DestinationFolder = outputFolderOverride; + } + + // set full path of output folder + _outputFolder = _fileService.GetFullPath(Path.Combine(_workingFolder, _config.DestinationFolder)); } + /// + /// Gets the list of files. This is a result from the RunAsync() method. + /// + public List Files { get; private set; } = []; + /// /// Run the action. /// @@ -47,23 +64,25 @@ public InventoryAction(string workingFolder, string configFile, string? outputFo public Task RunAsync() { ReturnCode ret = ReturnCode.Normal; + _logger.LogInformation($"\n*** INVENTORY STAGE."); try { - AssembleConfiguration config = ReadConfigurationAsync(_configFile); - if (!string.IsNullOrWhiteSpace(_outputFolder)) - { - // overwrite output folder with given value - config.OutputFolder = _outputFolder; - } + ret = GetAllFiles(); + ret &= ValidateFiles(); - foreach (var content in config.Content) + if (ret != ReturnCode.Error) { - var source = Path.Combine(_workingFolder, content.SourceFolder); - var files = _fileService.GetFiles(source, content.Files, content.Exclude); - foreach (var file in files) + ret &= UpdateLinks(); + + // log result of inventory (verbose) + foreach (var file in Files) { - _fileInfoService.GetExternalHyperlinks(source, file); + _logger.LogInformation($"{file.SourcePath} \n\t==> {file.DestinationPath}"); + foreach (var link in file.Links) + { + _logger.LogInformation($"\t{link.OriginalUrl} => {link.DestinationRelativeUrl ?? link.DestinationFullUrl}"); + } } } } @@ -73,9 +92,132 @@ public Task RunAsync() ret = ReturnCode.Error; } + _logger.LogInformation($"END OF INVENTORY STAGE. Result: {ret}"); return Task.FromResult(ret); } + private ReturnCode UpdateLinks() + { + ReturnCode ret = ReturnCode.Normal; + + foreach (var file in Files) + { + foreach (var link in file.Links) + { + var dest = Files.SingleOrDefault(x => x.SourcePath.Equals(link.UrlFullPath.NormalizePath(), StringComparison.Ordinal)); + if (dest != null) + { + // destination found. register and also (new) calculate relative path + link.DestinationFullUrl = dest.DestinationPath; + string dir = Path.GetDirectoryName(file.DestinationPath)!; + link.DestinationRelativeUrl = Path.GetRelativePath(dir, dest.DestinationPath).NormalizePath(); + if (!string.IsNullOrEmpty(link.UrlTopic)) + { + link.DestinationFullUrl += link.UrlTopic; + link.DestinationRelativeUrl += link.UrlTopic; + } + } + else + { + if (string.IsNullOrEmpty(file.ContentSet!.ExternalFilePrefix)) + { + // ERROR: no solution to fix this reference + _logger.LogCritical($"Error in a file reference. Link '{link.OriginalUrl}' in '{file.SourcePath}' cannot be resolved and no external file prefix was given."); + ret = ReturnCode.Error; + } + else + { + // we're calculating the link with the external file prefix, usualy a repo web link prefix. + string subpath = link.UrlFullPath.Substring(file.ContentSet!.SourceFolder.Length).TrimStart('/'); + link.DestinationFullUrl = file.ContentSet!.ExternalFilePrefix.TrimEnd('/') + "/" + subpath; + } + } + } + } + + return ret; + } + + private ReturnCode ValidateFiles() + { + ReturnCode ret = ReturnCode.Normal; + + var duplicates = Files.GroupBy(x => x.DestinationPath).Where(g => g.Count() > 1); + if (duplicates.Any()) + { + _logger.LogCritical("ERROR: one or more files will be overwritten. Validate content definitions. Consider exclude paths."); + foreach (var dup in duplicates) + { + _logger.LogCritical($"{dup.Key} used for:"); + foreach (var source in dup) + { + _logger.LogCritical($"\t{source.SourcePath} (Content group '{source.ContentSet!.SourceFolder}')"); + } + } + + ret = ReturnCode.Error; + } + + return ret; + } + + private ReturnCode GetAllFiles() + { + ReturnCode ret = ReturnCode.Normal; + + // loop through all content definitions + foreach (var content in _config.Content) + { + // determine source and destination folders + var sourceFolder = _fileService.GetFullPath(Path.Combine(_workingFolder, content.SourceFolder)); + var destFolder = _outputFolder!; + if (!string.IsNullOrEmpty(content.DestinationFolder)) + { + destFolder = _fileService.GetFullPath(Path.Combine(destFolder, content.DestinationFolder.Trim())); + } + + // get all files and loop through them to add to the this.Files collection + var files = _fileService.GetFiles(sourceFolder, content.Files, content.Exclude); + foreach (var file in files) + { + FileData fileData = new FileData + { + ContentSet = content, + SourcePath = file.NormalizePath(), + }; + + if (content.RawCopy != true && Path.GetExtension(file).Equals(".md", StringComparison.OrdinalIgnoreCase)) + { + // only for markdown, get the links + fileData.Links = _fileInfoService.GetLocalHyperlinks(sourceFolder, file); + } + + // set destination path of the file + string subpath = fileData.SourcePath.Substring(sourceFolder.Length).TrimStart('/'); + fileData.DestinationPath = _fileService.GetFullPath(Path.Combine(destFolder, subpath)); + + // if a replace pattern is defined, apply this to the destination path + if (content.ReplacePattern != null) + { + try + { + string replacement = content.ReplaceValue ?? string.Empty; + fileData.DestinationPath = Regex.Replace(fileData.DestinationPath, content.ReplacePattern, replacement); + } + catch (Exception ex) + { + _logger.LogError($"Regex error for source `{content.SourceFolder}`: {ex.Message}. No replacement done."); + ret = ReturnCode.Warning; + } + } + + Files.Add(fileData); + } + } + + return ret; + } + private AssembleConfiguration ReadConfigurationAsync(string configFile) { if (!_fileService.ExistsFileOrDirectory(configFile)) diff --git a/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs b/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs index b088f81..195ab2d 100644 --- a/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs +++ b/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs @@ -12,10 +12,10 @@ namespace DocAssembler.Configuration; public sealed record AssembleConfiguration { /// - /// Gets or sets the output folder. + /// Gets or sets the destination folder. /// - [JsonPropertyName("path")] - public string OutputFolder { get; set; } = string.Empty; + [JsonPropertyName("dest")] + public string DestinationFolder { get; set; } = string.Empty; /// /// Gets or sets the content to process. diff --git a/src/DocAssembler/DocAssembler/Configuration/Content.cs b/src/DocAssembler/DocAssembler/Configuration/Content.cs index 403e9c7..eb5d028 100644 --- a/src/DocAssembler/DocAssembler/Configuration/Content.cs +++ b/src/DocAssembler/DocAssembler/Configuration/Content.cs @@ -20,7 +20,7 @@ public sealed record Content /// /// Gets or sets the optional destination folder. /// - [JsonPropertyName("path")] + [JsonPropertyName("dest")] public string? DestinationFolder { get; set; } /// @@ -33,22 +33,30 @@ public sealed record Content /// Gets or sets the folders and files to exclude. /// This list supports the file glob pattern. /// - public List Exclude { get; set; } = new(); + public List? Exclude { get; set; } /// - /// Gets or sets naming strategy of the target folder. + /// Gets or sets the pattern to find in references to be replaced. This is a regex expression. + /// Works with the the to replace what was found. + /// Example: "\/docs\/". /// - public FolderNamingStrategy Naming { get; set; } + public string? ReplacePattern { get; set; } /// - /// Gets or sets a value indicating whether we need to fix markdown links. + /// Gets or sets the value to replace what was found with . /// - public bool FixMarkdownLinks { get; set; } = true; + public string? ReplaceValue { get; set; } + + /// + /// Gets or sets a value indicating whether we need to do just a raw copy. + /// + public bool? RawCopy { get; set; } /// /// Gets or sets the prefix for external files like source files. /// This is for all references to files that are not part of the - /// documentation hierarchy. Use this to prefix Gitlab URL's. + /// selected files (mostly markdown and assets). + /// An example use is to prefix the URL with the url of the github repo. /// - public string ExternalFilePrefix { get; set; } = string.Empty; + public string? ExternalFilePrefix { get; set; } } diff --git a/src/DocAssembler/DocAssembler/FileService/FileData.cs b/src/DocAssembler/DocAssembler/FileService/FileData.cs index f9063cf..1588783 100644 --- a/src/DocAssembler/DocAssembler/FileService/FileData.cs +++ b/src/DocAssembler/DocAssembler/FileService/FileData.cs @@ -2,6 +2,8 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using DocAssembler.Configuration; + namespace DocAssembler.FileService; /// @@ -10,12 +12,22 @@ namespace DocAssembler.FileService; public sealed record FileData { /// - /// Gets or sets the original path of the file. + /// Gets or sets the source full path of the file. + /// + public string SourcePath { get; set; } = string.Empty; + + /// + /// Gets or sets the destination full path of the file. + /// + public string DestinationPath { get; set; } = string.Empty; + + /// + /// Gets or sets the content set the file belongs to. /// - public string OriginalPath { get; set; } = string.Empty; + public Content? ContentSet { get; set; } /// - /// Gets or sets the output path of the file. + /// Gets or sets all links in the document we might need to work on. /// - public string OutputPath { get; set; } = string.Empty; + public List Links { get; set; } = []; } diff --git a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs index 16a6b9e..043f0de 100644 --- a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs +++ b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs @@ -32,12 +32,12 @@ public FileInfoService(string workingFolder, IFileService fileService, ILogger l } /// - /// Get the H1 title from a markdown file. + /// Get the local links in the markdown file. /// /// Root path of the documentation. /// File path of the markdown file. - /// First H1, or the filename as title if that fails. - public string GetExternalHyperlinks(string root, string filePath) + /// List of local links in the document. If none found, the list is empty. + public List GetLocalHyperlinks(string root, string filePath) { string markdownFilePath = _fileService.GetFullPath(filePath); string markdown = _fileService.ReadAllText(markdownFilePath); @@ -58,15 +58,14 @@ public string GetExternalHyperlinks(string root, string filePath) !x.Url.StartsWith("ftp://", StringComparison.InvariantCulture) && !x.Url.StartsWith("mailto:", StringComparison.InvariantCulture) && !x.Url.StartsWith("xref:", StringComparison.InvariantCulture)) + .Select(d => new Hyperlink(markdownFilePath, d.Line + 1, d.Column + 1, d.Url ?? string.Empty) + { + UrlSpanStart = d.UrlSpan.Start, + UrlSpanEnd = d.UrlSpan.End, + UrlSpanLength = d.UrlSpan.Length, + }) .ToList(); - foreach (var link in links) - { - string url = _fileService.GetFullPath(Path.Combine(_workingFolder, link.Url!)); - Console.WriteLine($"{markdown.Substring(link.Span.Start, link.Span.Length)} => {url}"); - } - - // in case we couldn't get an H1 from markdown, return the filepath sanitized. - return $"Hello markdown!"; + return links; } } diff --git a/src/DocAssembler/DocAssembler/FileService/FilePathExtensions.cs b/src/DocAssembler/DocAssembler/FileService/FilePathExtensions.cs new file mode 100644 index 0000000..d236af1 --- /dev/null +++ b/src/DocAssembler/DocAssembler/FileService/FilePathExtensions.cs @@ -0,0 +1,36 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using System.Diagnostics.CodeAnalysis; + +namespace DocAssembler.FileService; + +/// +/// File path extension methods. +/// +[ExcludeFromCodeCoverage] +public static class FilePathExtensions +{ + /// + /// Normalize the path to have a common notation of directory separators. + /// This is needed when used in Equal() methods and such. + /// + /// Path to normalize. + /// Normalized path. + public static string NormalizePath(this string path) + { + return path.Replace("\\", "/"); + } + + /// + /// Normalize the content. This is used to make sure we always + /// have "\n" only for new lines. Mostly used by the test mocks. + /// + /// Content to normalize. + /// Normalized content. + public static string NormalizeContent(this string content) + { + return content.Replace("\r", string.Empty); + } +} diff --git a/src/DocAssembler/DocAssembler/FileService/FileService.cs b/src/DocAssembler/DocAssembler/FileService/FileService.cs index ecce860..9741f55 100644 --- a/src/DocAssembler/DocAssembler/FileService/FileService.cs +++ b/src/DocAssembler/DocAssembler/FileService/FileService.cs @@ -16,7 +16,7 @@ public class FileService : IFileService /// public string GetFullPath(string path) { - return Path.GetFullPath(path); + return Path.GetFullPath(path).NormalizePath(); } /// @@ -26,7 +26,7 @@ public bool ExistsFileOrDirectory(string path) } /// - public IEnumerable GetFiles(string root, List includes, List excludes) + public IEnumerable GetFiles(string root, List includes, List? excludes) { string fullRoot = Path.GetFullPath(root); Matcher matcher = new(); @@ -35,14 +35,17 @@ public IEnumerable GetFiles(string root, List includes, List x.Replace("\\", "/")) + .Select(x => x.NormalizePath()) .ToList(); } @@ -75,4 +78,11 @@ public Stream OpenRead(string path) { return File.OpenRead(path); } + + /// + public void Copy(string source, string destination) + { + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); + File.Copy(source, destination); + } } diff --git a/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs b/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs new file mode 100644 index 0000000..69e3d9e --- /dev/null +++ b/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs @@ -0,0 +1,298 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +using System.Text; + +namespace DocAssembler.FileService; + +/// +/// Hyperlink in document. +/// +public class Hyperlink +{ + private static readonly char[] UriFragmentOrQueryString = new char[] { '#', '?' }; + private static readonly char[] AdditionalInvalidChars = @"\/?:*".ToArray(); + private static readonly char[] InvalidPathChars = Path.GetInvalidPathChars().Concat(AdditionalInvalidChars).ToArray(); + + /// + /// Initializes a new instance of the class. + /// + public Hyperlink() + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Path of the markdown file. + /// Line number. + /// Column. + /// Url. + public Hyperlink(string filePath, int line, int col, string url) + { + FilePath = filePath; + Line = line; + Column = col; + + Url = url; + OriginalUrl = Url; + + LinkType = HyperlinkType.Empty; + if (!string.IsNullOrWhiteSpace(url)) + { + if (url.StartsWith("https://", StringComparison.InvariantCulture) || url.StartsWith("http://", StringComparison.InvariantCulture)) + { + LinkType = HyperlinkType.Webpage; + } + else if (url.StartsWith("ftps://", StringComparison.InvariantCulture) || url.StartsWith("ftp://", StringComparison.InvariantCulture)) + { + LinkType = HyperlinkType.Ftp; + } + else if (url.StartsWith("mailto:", StringComparison.InvariantCulture)) + { + LinkType = HyperlinkType.Mail; + } + else if (url.StartsWith("xref:", StringComparison.InvariantCulture)) + { + LinkType = HyperlinkType.CrossReference; + } + else + { + Url = UrlDecode(Url); + + if (Path.GetExtension(url).Equals(".md", StringComparison.OrdinalIgnoreCase) || Path.GetExtension(url) == string.Empty) + { + // link to an MD file or a folder + LinkType = HyperlinkType.Local; + } + else + { + // link to image or something like that. + LinkType = HyperlinkType.Resource; + } + } + } + } + + /// + /// Gets or sets the file path name of the markdown file. + /// + public string FilePath { get; set; } = string.Empty; + + /// + /// Gets or sets the line number in the file. + /// + public int Line { get; set; } + + /// + /// Gets or sets the column in the file. + /// + public int Column { get; set; } + + /// + /// Gets or sets the URL span start. + /// + public int UrlSpanStart { get; set; } + + /// + /// Gets or sets the URL span end. This might differ from Markdig span end, + /// as we're trimming any #-reference at the end. + /// + public int UrlSpanEnd { get; set; } + + /// + /// Gets or sets the URL span length. This might differ from Markdig span length, + /// as we're trimming any #-reference at the end. + /// + public int UrlSpanLength { get; set; } + + /// + /// Gets or sets the URL. + /// + public string Url { get; set; } = string.Empty; + + /// + /// Gets or sets the original URL as found in the Markdown document. Used for reporting to user so they can find the correct location. Url will be modified. + /// + public string OriginalUrl { get; set; } = string.Empty; + + /// + /// Gets or sets the full destination url. + /// + public string? DestinationFullUrl { get; set; } + + /// + /// Gets or sets the relative destination url. + /// + public string? DestinationRelativeUrl { get; set; } + + /// + /// Gets or sets a value indicating whether this is a web link. + /// + public HyperlinkType LinkType { get; set; } + + /// + /// Gets a value indicating whether this is a local link. + /// + public bool IsLocal + { + get + { + return LinkType == HyperlinkType.Local || LinkType == HyperlinkType.Resource; + } + } + + /// + /// Gets a value indicating whether this is a web link. + /// + public bool IsWeb + { + get + { + return LinkType == HyperlinkType.Webpage || LinkType == HyperlinkType.Ftp; + } + } + + /// + /// Gets the topic in the url. This is the id after the # in a local link. + /// Otherwise it's returned empty. + /// + public string UrlTopic + { + get + { + if (IsLocal) + { + int pos = Url.IndexOf('#', StringComparison.InvariantCulture); + if (pos == -1) + { + // if we don't have a header delimiter, we might have a url delimiter + pos = Url.IndexOf('?', StringComparison.InvariantCulture); + } + + // include the separator + return pos == -1 ? string.Empty : Url.Substring(pos); + } + + return string.Empty; + } + } + + /// + /// Gets the url without a (possible) topic. This is the id after the # in a local link. + /// + public string UrlWithoutTopic + { + get + { + if (IsLocal) + { + int pos = Url.IndexOf('#', StringComparison.InvariantCulture); + if (pos == -1) + { + // if we don't have a header delimiter, we might have a url delimiter + pos = Url.IndexOf('?', StringComparison.InvariantCulture); + } + + switch (pos) + { + case -1: + return Url; + case 0: + return FilePath; + default: + return Url.Substring(0, pos); + } + } + + return Url; + } + } + + /// + /// Gets the full path of the URL when it's a local reference, but without the topic. + /// It's calculated relative to the file path. + /// + public string UrlFullPath + { + get + { + if (IsLocal) + { + int pos = Url.IndexOf('#', StringComparison.InvariantCulture); + if (pos == -1) + { + // if we don't have a header delimiter, we might have a url delimiter + pos = Url.IndexOf('?', StringComparison.InvariantCulture); + } + + // we want to know that the link is not starting with a # for local reference. + // if local reference, return the filename otherwise the calculated path. + string destFullPath = pos != 0 ? + Path.Combine(Path.GetDirectoryName(FilePath)!, UrlWithoutTopic) : + FilePath; + return Path.GetFullPath(destFullPath).NormalizePath(); + } + + return Url; + } + } + + /// + /// Decoding of local Urls. Similar to logic from DocFx RelativePath class. + /// https://github.com/dotnet/docfx/blob/cca05f505e30c5ede36973c4b989fce711f2e8ad/src/Docfx.Common/Path/RelativePath.cs . + /// + /// Url. + /// Decoded Url. + private string UrlDecode(string url) + { + // This logic only applies to relative paths. + if (Path.IsPathRooted(url)) + { + return url; + } + + var anchor = string.Empty; + var index = url.IndexOfAny(UriFragmentOrQueryString); + if (index != -1) + { + anchor = url.Substring(index); + url = url.Remove(index); + } + + var parts = url.Split('/', '\\'); + var newUrl = new StringBuilder(); + for (int i = 0; i < parts.Length; i++) + { + if (i > 0) + { + newUrl.Append('/'); + } + + var origin = parts[i]; + var value = Uri.UnescapeDataString(origin); + + var splittedOnInvalidChars = value.Split(InvalidPathChars); + var originIndex = 0; + var valueIndex = 0; + for (int j = 0; j < splittedOnInvalidChars.Length; j++) + { + if (j > 0) + { + var invalidChar = value[valueIndex]; + valueIndex++; + newUrl.Append(Uri.EscapeDataString(invalidChar.ToString())); + } + + var splitOnInvalidChars = splittedOnInvalidChars[j]; + originIndex += splitOnInvalidChars.Length; + valueIndex += splitOnInvalidChars.Length; + newUrl.Append(splitOnInvalidChars); + } + } + + newUrl.Append(anchor); + return newUrl.ToString(); + } +} diff --git a/src/DocAssembler/DocAssembler/FileService/HyperlinkType.cs b/src/DocAssembler/DocAssembler/FileService/HyperlinkType.cs new file mode 100644 index 0000000..48d1bb9 --- /dev/null +++ b/src/DocAssembler/DocAssembler/FileService/HyperlinkType.cs @@ -0,0 +1,47 @@ +namespace DocAssembler.FileService; + +/// +/// Enumeration of hyperlink types. +/// +public enum HyperlinkType +{ + /// + /// Local file. + /// + Local, + + /// + /// A web page (http or https). + /// + Webpage, + + /// + /// A download link (ftp or ftps). + /// + Ftp, + + /// + /// Mail address (mailto). + /// + Mail, + + /// + /// A cross reference (xref). + /// + CrossReference, + + /// + /// A local resource, like an image. + /// + Resource, + + /// + /// A tab - DocFx special. See https://dotnet.github.io/docfx/docs/markdown.html?tabs=linux%2Cdotnet#tabs. + /// + Tab, + + /// + /// Empty link. + /// + Empty, +} diff --git a/src/DocAssembler/DocAssembler/FileService/IFileService.cs b/src/DocAssembler/DocAssembler/FileService/IFileService.cs index 6e2b1be..69f29d7 100644 --- a/src/DocAssembler/DocAssembler/FileService/IFileService.cs +++ b/src/DocAssembler/DocAssembler/FileService/IFileService.cs @@ -31,7 +31,7 @@ public interface IFileService /// Include patterns. /// Exclude patterns. /// List of files. - IEnumerable GetFiles(string root, List includes, List excludes); + IEnumerable GetFiles(string root, List includes, List? excludes); /// /// Get directories in the given path. @@ -67,4 +67,11 @@ public interface IFileService /// Path of the file. /// A . Stream OpenRead(string path); + + /// + /// Copy the given file to the destination. + /// + /// Source file path. + /// Destination file path. + void Copy(string source, string destination); } diff --git a/src/DocAssembler/DocAssembler/Program.cs b/src/DocAssembler/DocAssembler/Program.cs index 6fb9e1d..81e9dc8 100644 --- a/src/DocAssembler/DocAssembler/Program.cs +++ b/src/DocAssembler/DocAssembler/Program.cs @@ -125,7 +125,7 @@ async Task AssembleDocumentationAsync( InventoryAction inventory = new(folder, configFile.FullName, outputFolder?.FullName, fileService, logger); ret &= await inventory.RunAsync(); - AssembleAction assemble = new(configFile.FullName, outputFolder?.FullName, fileService, logger); + AssembleAction assemble = new(inventory.Files, fileService, logger); ret &= await assemble.RunAsync(); logger.LogInformation($"Command completed. Return value: {ret}."); From ff70557be9f9d604f3f8fb810617b416adf2fddf Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Wed, 27 Nov 2024 10:47:40 +0100 Subject: [PATCH 04/12] WIP: DocAssembler is getting there. Adding extra config --- .../DocAssembler/Actions/AssembleAction.cs | 1 - .../DocAssembler/Actions/InventoryAction.cs | 31 ++------- .../DocAssembler/DocAssembler.csproj | 6 +- .../FileService/FileInfoService.cs | 35 +++++++++++ .../DocAssembler/FileService/FileService.cs | 9 +++ .../DocAssembler/FileService/Hyperlink.cs | 36 ++--------- .../DocAssembler/FileService/IFileService.cs | 6 ++ src/DocAssembler/DocAssembler/Program.cs | 63 +++++++++++++++---- 8 files changed, 117 insertions(+), 70 deletions(-) diff --git a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs index 691a395..1dadd99 100644 --- a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs @@ -2,7 +2,6 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // -using System.Diagnostics; using System.Text; using DocAssembler.FileService; using Microsoft.Extensions.Logging; diff --git a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs index 94ff343..ecac11b 100644 --- a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs @@ -2,6 +2,7 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using System.Diagnostics; using System.Text.RegularExpressions; using DocAssembler.Configuration; using DocAssembler.FileService; @@ -16,8 +17,7 @@ namespace DocAssembler.Actions; public class InventoryAction { private readonly string _workingFolder; - private readonly string _configFile; - private readonly string? _outputFolder; + private readonly string _outputFolder; private readonly FileInfoService _fileInfoService; private readonly IFileService _fileService; private readonly ILogger _logger; @@ -28,26 +28,18 @@ public class InventoryAction /// Initializes a new instance of the class. /// /// Working folder. - /// Configuration file path. - /// Output folder override. + /// Configuration. /// File service. /// Logger. - public InventoryAction(string workingFolder, string configFile, string? outputFolderOverride, IFileService fileService, ILogger logger) + public InventoryAction(string workingFolder, AssembleConfiguration config, IFileService fileService, ILogger logger) { _workingFolder = workingFolder; - _configFile = configFile; + _config = config; _fileService = fileService; _logger = logger; _fileInfoService = new(workingFolder, _fileService, _logger); - _config = ReadConfigurationAsync(_configFile); - if (!string.IsNullOrWhiteSpace(outputFolderOverride)) - { - // overwrite output folder with given override value - _config.DestinationFolder = outputFolderOverride; - } - // set full path of output folder _outputFolder = _fileService.GetFullPath(Path.Combine(_workingFolder, _config.DestinationFolder)); } @@ -104,7 +96,7 @@ private ReturnCode UpdateLinks() { foreach (var link in file.Links) { - var dest = Files.SingleOrDefault(x => x.SourcePath.Equals(link.UrlFullPath.NormalizePath(), StringComparison.Ordinal)); + var dest = Files.SingleOrDefault(x => x.SourcePath.Equals(link.UrlFullPath, StringComparison.Ordinal)); if (dest != null) { // destination found. register and also (new) calculate relative path @@ -217,15 +209,4 @@ private ReturnCode GetAllFiles() return ret; } - - private AssembleConfiguration ReadConfigurationAsync(string configFile) - { - if (!_fileService.ExistsFileOrDirectory(configFile)) - { - throw new ActionException($"Configuration file '{configFile}' doesn't exist."); - } - - string json = _fileService.ReadAllText(configFile); - return SerializationUtil.Deserialize(json); - } } diff --git a/src/DocAssembler/DocAssembler/DocAssembler.csproj b/src/DocAssembler/DocAssembler/DocAssembler.csproj index eb5989f..5ada01e 100644 --- a/src/DocAssembler/DocAssembler/DocAssembler.csproj +++ b/src/DocAssembler/DocAssembler/DocAssembler.csproj @@ -32,9 +32,9 @@ - - - + + + diff --git a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs index 043f0de..e5a901e 100644 --- a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs +++ b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs @@ -2,6 +2,8 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using System; +using System.Diagnostics; using Markdig; using Markdig.Syntax; using Markdig.Syntax.Inlines; @@ -66,6 +68,39 @@ public List GetLocalHyperlinks(string root, string filePath) }) .ToList(); + foreach (var link in links) + { + if (link.Url != null && !link.Url.Equals(markdown.Substring(link.UrlSpanStart, link.UrlSpanLength), StringComparison.Ordinal)) + { + // MARKDIG FIX + // In some cases the Url in MarkDig LinkInline is not equal to the original + // e.g. a link "..\..\somefile.md" resolves in "....\somefile.md" + // we fix that here. This will probably not be fixed in the markdig + // library, as you shouldn't use backslash, but Unix-style slash. + link.Url = markdown.Substring(link.UrlSpanStart, link.UrlSpanLength); + } + + if (link.IsLocal) + { + int pos = link.Url!.IndexOf('#', StringComparison.InvariantCulture); + if (pos == -1) + { + // if we don't have a header delimiter, we might have a url delimiter + pos = link.Url.IndexOf('?', StringComparison.InvariantCulture); + } + + // we want to know that the link is not starting with a # for local reference. + // if local reference, return the filename otherwise the calculated path. + string destFullPath = pos != 0 ? + Path.Combine(Path.GetDirectoryName(link.FilePath)!, link.UrlWithoutTopic) : link.FilePath; + link.UrlFullPath = _fileService.GetFullPath(destFullPath); + } + else + { + link.UrlFullPath = link.Url!; + } + } + return links; } } diff --git a/src/DocAssembler/DocAssembler/FileService/FileService.cs b/src/DocAssembler/DocAssembler/FileService/FileService.cs index 9741f55..b0dd192 100644 --- a/src/DocAssembler/DocAssembler/FileService/FileService.cs +++ b/src/DocAssembler/DocAssembler/FileService/FileService.cs @@ -85,4 +85,13 @@ public void Copy(string source, string destination) Directory.CreateDirectory(Path.GetDirectoryName(destination)!); File.Copy(source, destination); } + + /// + public void DeleteFolder(string path) + { + if (Directory.Exists(path)) + { + Directory.Delete(path); + } + } } diff --git a/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs b/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs index 69e3d9e..e919597 100644 --- a/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs +++ b/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs @@ -59,7 +59,7 @@ public Hyperlink(string filePath, int line, int col, string url) } else { - Url = UrlDecode(Url); + Url = UrlDecode(Url).NormalizePath(); if (Path.GetExtension(url).Equals(".md", StringComparison.OrdinalIgnoreCase) || Path.GetExtension(url) == string.Empty) { @@ -112,6 +112,11 @@ public Hyperlink(string filePath, int line, int col, string url) /// public string Url { get; set; } = string.Empty; + /// + /// Gets or sets the URL full path. + /// + public string UrlFullPath { get; set; } = string.Empty; + /// /// Gets or sets the original URL as found in the Markdown document. Used for reporting to user so they can find the correct location. Url will be modified. /// @@ -210,35 +215,6 @@ public string UrlWithoutTopic } } - /// - /// Gets the full path of the URL when it's a local reference, but without the topic. - /// It's calculated relative to the file path. - /// - public string UrlFullPath - { - get - { - if (IsLocal) - { - int pos = Url.IndexOf('#', StringComparison.InvariantCulture); - if (pos == -1) - { - // if we don't have a header delimiter, we might have a url delimiter - pos = Url.IndexOf('?', StringComparison.InvariantCulture); - } - - // we want to know that the link is not starting with a # for local reference. - // if local reference, return the filename otherwise the calculated path. - string destFullPath = pos != 0 ? - Path.Combine(Path.GetDirectoryName(FilePath)!, UrlWithoutTopic) : - FilePath; - return Path.GetFullPath(destFullPath).NormalizePath(); - } - - return Url; - } - } - /// /// Decoding of local Urls. Similar to logic from DocFx RelativePath class. /// https://github.com/dotnet/docfx/blob/cca05f505e30c5ede36973c4b989fce711f2e8ad/src/Docfx.Common/Path/RelativePath.cs . diff --git a/src/DocAssembler/DocAssembler/FileService/IFileService.cs b/src/DocAssembler/DocAssembler/FileService/IFileService.cs index 69f29d7..37ec990 100644 --- a/src/DocAssembler/DocAssembler/FileService/IFileService.cs +++ b/src/DocAssembler/DocAssembler/FileService/IFileService.cs @@ -74,4 +74,10 @@ public interface IFileService /// Source file path. /// Destination file path. void Copy(string source, string destination); + + /// + /// Delete given folder path. + /// + /// Path of the folder. + void DeleteFolder(string path); } diff --git a/src/DocAssembler/DocAssembler/Program.cs b/src/DocAssembler/DocAssembler/Program.cs index 81e9dc8..19f5d81 100644 --- a/src/DocAssembler/DocAssembler/Program.cs +++ b/src/DocAssembler/DocAssembler/Program.cs @@ -7,6 +7,7 @@ using System.CommandLine.Parsing; using DocAssembler; using DocAssembler.Actions; +using DocAssembler.Configuration; using DocAssembler.FileService; using DocAssembler.Utils; using Microsoft.Extensions.Logging; @@ -29,6 +30,10 @@ name: "--outfolder", description: "Override the output folder for the assembled documentation in the config file."); +var cleanupOption = new Option( + name: "--cleanup-output", + description: "Cleanup the output folder before generating. NOTE: This will delete all folders and files!"); + var verboseOption = new Option( name: "--verbose", description: "Show verbose messages of the process."); @@ -49,6 +54,7 @@ Assemble documentation in the output folder. The tool will also fix links follow rootCommand.AddOption(workingFolderOption); rootCommand.AddOption(configFileOption); rootCommand.AddOption(outputFolderOption); +rootCommand.AddOption(cleanupOption); rootCommand.AddOption(verboseOption); var initCommand = new Command("init", "Intialize a configuration file in the current directory if it doesn't exist yet."); @@ -63,13 +69,15 @@ Assemble documentation in the output folder. The tool will also fix links follow LogParameters( context.ParseResult.GetValueForOption(configFileOption)!, context.ParseResult.GetValueForOption(outputFolderOption), - context.ParseResult.GetValueForOption(workingFolderOption)); + context.ParseResult.GetValueForOption(workingFolderOption), + context.ParseResult.GetValueForOption(cleanupOption)); // execute the generator context.ExitCode = (int)await AssembleDocumentationAsync( context.ParseResult.GetValueForOption(configFileOption)!, context.ParseResult.GetValueForOption(outputFolderOption), - context.ParseResult.GetValueForOption(workingFolderOption)); + context.ParseResult.GetValueForOption(workingFolderOption), + context.ParseResult.GetValueForOption(cleanupOption)); }); // handle the execution of the root command @@ -111,7 +119,8 @@ async Task GenerateConfigurationFile() async Task AssembleDocumentationAsync( FileInfo configFile, DirectoryInfo? outputFolder, - DirectoryInfo? workingFolder) + DirectoryInfo? workingFolder, + bool cleanup) { // setup services ILogger logger = GetLogger(); @@ -121,10 +130,41 @@ async Task AssembleDocumentationAsync( { ReturnCode ret = ReturnCode.Normal; - string folder = workingFolder?.FullName ?? Directory.GetCurrentDirectory(); - InventoryAction inventory = new(folder, configFile.FullName, outputFolder?.FullName, fileService, logger); + string currentFolder = workingFolder?.FullName ?? Directory.GetCurrentDirectory(); + + // CONFIGURATION + if (!Path.Exists(configFile.FullName)) + { + // error: not found + logger.LogCritical($"Configuration file '{configFile}' doesn't exist."); + return ReturnCode.Error; + } + + string json = File.ReadAllText(configFile.FullName); + var config = SerializationUtil.Deserialize(json); + string outputFolderPath = string.Empty; + if (outputFolder != null) + { + // overwrite output folder with given override value + config.DestinationFolder = outputFolder.FullName; + outputFolderPath = outputFolder.FullName; + } + else + { + outputFolderPath = Path.GetFullPath(Path.Combine(currentFolder, config.DestinationFolder)); + } + + // INVENTORY + InventoryAction inventory = new(currentFolder, config, fileService, logger); ret &= await inventory.RunAsync(); + if (cleanup && Directory.Exists(outputFolderPath)) + { + // CLEANUP OUTPUT + Directory.Delete(outputFolderPath, true); + } + + // ASSEMBLE AssembleAction assemble = new(inventory.Files, fileService, logger); ret &= await assemble.RunAsync(); @@ -143,22 +183,23 @@ async Task AssembleDocumentationAsync( void LogParameters( FileInfo configFile, DirectoryInfo? outputFolder, - DirectoryInfo? workingFolder) + DirectoryInfo? workingFolder, + bool cleanup) { ILogger logger = GetLogger(); - logger!.LogInformation($"Configuration : {configFile.FullName}"); + logger.LogInformation($"Configuration : {configFile.FullName}"); if (outputFolder != null) { - logger!.LogInformation($"Output folder: {outputFolder.FullName}"); - return; + logger.LogInformation($"Output folder: {outputFolder.FullName}"); } if (workingFolder != null) { - logger!.LogInformation($"Working folder: {workingFolder.FullName}"); - return; + logger.LogInformation($"Working folder: {workingFolder.FullName}"); } + + logger.LogInformation($"Cleanup : {cleanup}"); } void SetLogLevel(InvocationContext context) From b9bb314841668ba8c19f6f558b31cbe1d9e24e39 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Wed, 27 Nov 2024 18:05:27 +0100 Subject: [PATCH 05/12] WIP: tested some edge cases --- .../DocAssembler/Actions/AssembleAction.cs | 30 ++++++-- .../DocAssembler/Actions/ConfigInitAction.cs | 10 ++- .../DocAssembler/Actions/InventoryAction.cs | 68 +++++++++++-------- .../DocAssembler/Configuration/Content.cs | 10 ++- .../DocAssembler/Configuration/Replacement.cs | 21 ++++++ .../DocAssembler/FileService/FileData.cs | 5 ++ .../FileService/FileInfoService.cs | 9 ++- src/DocAssembler/DocAssembler/Program.cs | 19 +++--- 8 files changed, 123 insertions(+), 49 deletions(-) create mode 100644 src/DocAssembler/DocAssembler/Configuration/Replacement.cs diff --git a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs index 1dadd99..b46d47e 100644 --- a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs @@ -3,6 +3,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. // using System.Text; +using System.Text.RegularExpressions; using DocAssembler.FileService; using Microsoft.Extensions.Logging; @@ -50,7 +51,7 @@ public Task RunAsync() var updates = file.Links .Where(x => !x.OriginalUrl.Equals(x.DestinationRelativeUrl ?? x.DestinationFullUrl, StringComparison.Ordinal)) .OrderBy(x => x.UrlSpanStart); - if (updates.Any()) + if (file.IsMarkdown && (updates.Any() || file.ContentSet?.ContentReplacements is not null)) { var markdown = _fileService.ReadAllText(file.SourcePath); StringBuilder sb = new StringBuilder(); @@ -58,7 +59,6 @@ public Task RunAsync() foreach (var update in updates) { // first append text so far from markdown - Console.WriteLine($"pos={pos} len={update.UrlSpanStart - pos} md={markdown.Length}"); sb.Append(markdown.AsSpan(pos, update.UrlSpanStart - pos)); // append new link @@ -70,10 +70,32 @@ public Task RunAsync() // add final part of markdown sb.Append(markdown.AsSpan(pos)); + string output = sb.ToString(); + + // if replacement patterns are defined, apply them to the content + int replacements = 0; + if (file.ContentSet?.ContentReplacements is not null) + { + try + { + // apply all replacements + foreach (var replacement in file.ContentSet.ContentReplacements) + { + string r = replacement.Value ?? string.Empty; + output = Regex.Replace(output, replacement.Expression, r); + replacements++; + } + } + catch (Exception ex) + { + _logger.LogError($"Regex error for source `{file.SourcePath}`: {ex.Message}. No replacement done."); + ret = ReturnCode.Warning; + } + } Directory.CreateDirectory(Path.GetDirectoryName(file.DestinationPath)!); - _fileService.WriteAllText(file.DestinationPath, sb.ToString()); - _logger.LogInformation($"Copied '{file.SourcePath}' to '{file.DestinationPath}'. Replace {updates.Count()} links."); + _fileService.WriteAllText(file.DestinationPath, output); + _logger.LogInformation($"Copied '{file.SourcePath}' to '{file.DestinationPath}' with {updates.Count()} URL replacements and {replacements} content replacements."); } else { diff --git a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs index fed5e34..207264b 100644 --- a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs @@ -2,6 +2,7 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using System.Linq.Expressions; using DocAssembler.Configuration; using DocAssembler.FileService; using DocAssembler.Utils; @@ -79,8 +80,13 @@ public async Task RunAsync() SourceFolder = "backend", DestinationFolder = "services", Files = { "**/docs/**" }, - ReplacePattern = "/[Dd]ocs/", - ReplaceValue = "/", + UrlReplacements = [ + new Replacement + { + Expression = "/[Dd]ocs/", + Value = "/", + } + ], ExternalFilePrefix = "https://github.com/example/blob/main/", }, ], diff --git a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs index ecac11b..bfbcdd2 100644 --- a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs @@ -80,7 +80,7 @@ public Task RunAsync() } catch (Exception ex) { - _logger.LogCritical($"Reading configuration error: {ex.Message}"); + _logger.LogCritical($"Inventory error: {ex.Message}"); ret = ReturnCode.Error; } @@ -94,34 +94,39 @@ private ReturnCode UpdateLinks() foreach (var file in Files) { - foreach (var link in file.Links) + if (file.Links.Count > 0) { - var dest = Files.SingleOrDefault(x => x.SourcePath.Equals(link.UrlFullPath, StringComparison.Ordinal)); - if (dest != null) - { - // destination found. register and also (new) calculate relative path - link.DestinationFullUrl = dest.DestinationPath; - string dir = Path.GetDirectoryName(file.DestinationPath)!; - link.DestinationRelativeUrl = Path.GetRelativePath(dir, dest.DestinationPath).NormalizePath(); - if (!string.IsNullOrEmpty(link.UrlTopic)) - { - link.DestinationFullUrl += link.UrlTopic; - link.DestinationRelativeUrl += link.UrlTopic; - } - } - else + _logger.LogInformation($"Updating links for '{file.SourcePath}'"); + + foreach (var link in file.Links) { - if (string.IsNullOrEmpty(file.ContentSet!.ExternalFilePrefix)) + var dest = Files.SingleOrDefault(x => x.SourcePath.Equals(link.UrlFullPath, StringComparison.Ordinal)); + if (dest != null) { - // ERROR: no solution to fix this reference - _logger.LogCritical($"Error in a file reference. Link '{link.OriginalUrl}' in '{file.SourcePath}' cannot be resolved and no external file prefix was given."); - ret = ReturnCode.Error; + // destination found. register and also (new) calculate relative path + link.DestinationFullUrl = dest.DestinationPath; + string dir = Path.GetDirectoryName(file.DestinationPath)!; + link.DestinationRelativeUrl = Path.GetRelativePath(dir, dest.DestinationPath).NormalizePath(); + if (!string.IsNullOrEmpty(link.UrlTopic)) + { + link.DestinationFullUrl += link.UrlTopic; + link.DestinationRelativeUrl += link.UrlTopic; + } } else { - // we're calculating the link with the external file prefix, usualy a repo web link prefix. - string subpath = link.UrlFullPath.Substring(file.ContentSet!.SourceFolder.Length).TrimStart('/'); - link.DestinationFullUrl = file.ContentSet!.ExternalFilePrefix.TrimEnd('/') + "/" + subpath; + if (string.IsNullOrEmpty(file.ContentSet!.ExternalFilePrefix)) + { + // ERROR: no solution to fix this reference + _logger.LogCritical($"Error in a file reference. Link '{link.OriginalUrl}' in '{file.SourcePath}' cannot be resolved and no external file prefix was given."); + ret = ReturnCode.Error; + } + else + { + // we're calculating the link with the external file prefix, usualy a repo web link prefix. + string subpath = link.UrlFullPath.Substring(file.ContentSet!.SourceFolder.Length).TrimStart('/'); + link.DestinationFullUrl = file.ContentSet!.ExternalFilePrefix.TrimEnd('/') + "/" + subpath; + } } } } @@ -168,10 +173,13 @@ private ReturnCode GetAllFiles() destFolder = _fileService.GetFullPath(Path.Combine(destFolder, content.DestinationFolder.Trim())); } + _logger.LogInformation($"Processing content for '{sourceFolder}' => '{destFolder}'"); + // get all files and loop through them to add to the this.Files collection var files = _fileService.GetFiles(sourceFolder, content.Files, content.Exclude); foreach (var file in files) { + _logger.LogInformation($"- '{file}'"); FileData fileData = new FileData { ContentSet = content, @@ -188,17 +196,21 @@ private ReturnCode GetAllFiles() string subpath = fileData.SourcePath.Substring(sourceFolder.Length).TrimStart('/'); fileData.DestinationPath = _fileService.GetFullPath(Path.Combine(destFolder, subpath)); - // if a replace pattern is defined, apply this to the destination path - if (content.ReplacePattern != null) + // if replace patterns are defined, apply them to the destination path + if (content.UrlReplacements != null) { try { - string replacement = content.ReplaceValue ?? string.Empty; - fileData.DestinationPath = Regex.Replace(fileData.DestinationPath, content.ReplacePattern, replacement); + // apply all replacements + foreach (var replacement in content.UrlReplacements) + { + string r = replacement.Value ?? string.Empty; + fileData.DestinationPath = Regex.Replace(fileData.DestinationPath, replacement.Expression, r); + } } catch (Exception ex) { - _logger.LogError($"Regex error for source `{content.SourceFolder}`: {ex.Message}. No replacement done."); + _logger.LogError($"Regex error for file `{file}`: {ex.Message}. No replacement done."); ret = ReturnCode.Warning; } } diff --git a/src/DocAssembler/DocAssembler/Configuration/Content.cs b/src/DocAssembler/DocAssembler/Configuration/Content.cs index eb5d028..f542cdc 100644 --- a/src/DocAssembler/DocAssembler/Configuration/Content.cs +++ b/src/DocAssembler/DocAssembler/Configuration/Content.cs @@ -36,16 +36,14 @@ public sealed record Content public List? Exclude { get; set; } /// - /// Gets or sets the pattern to find in references to be replaced. This is a regex expression. - /// Works with the the to replace what was found. - /// Example: "\/docs\/". + /// Gets or sets the URL replacements. /// - public string? ReplacePattern { get; set; } + public List? UrlReplacements { get; set; } /// - /// Gets or sets the value to replace what was found with . + /// Gets or sets the content replacements. /// - public string? ReplaceValue { get; set; } + public List? ContentReplacements { get; set; } /// /// Gets or sets a value indicating whether we need to do just a raw copy. diff --git a/src/DocAssembler/DocAssembler/Configuration/Replacement.cs b/src/DocAssembler/DocAssembler/Configuration/Replacement.cs new file mode 100644 index 0000000..d9bbaba --- /dev/null +++ b/src/DocAssembler/DocAssembler/Configuration/Replacement.cs @@ -0,0 +1,21 @@ +// +// Copyright (c) DocFx Companion Tools. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// +namespace DocAssembler.Configuration; + +/// +/// Replacement definition. +/// +public sealed record Replacement +{ + /// + /// Gets or sets the regex expression for the replacement. + /// + public string Expression { get; set; } = string.Empty; + + /// + /// Gets or sets the replacement value. + /// + public string? Value { get; set; } +} diff --git a/src/DocAssembler/DocAssembler/FileService/FileData.cs b/src/DocAssembler/DocAssembler/FileService/FileData.cs index 1588783..da20ab1 100644 --- a/src/DocAssembler/DocAssembler/FileService/FileData.cs +++ b/src/DocAssembler/DocAssembler/FileService/FileData.cs @@ -30,4 +30,9 @@ public sealed record FileData /// Gets or sets all links in the document we might need to work on. /// public List Links { get; set; } = []; + + /// + /// Gets a value indicating whether the file is a markdown file. + /// + public bool IsMarkdown => Path.GetExtension(SourcePath).Equals(".md", StringComparison.OrdinalIgnoreCase); } diff --git a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs index e5a901e..b4832c0 100644 --- a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs +++ b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs @@ -53,7 +53,7 @@ public List GetLocalHyperlinks(string root, string filePath) var links = document .Descendants() .Where(x => !x.UrlHasPointyBrackets && - x.Url != null && + !string.IsNullOrEmpty(x.Url) && !x.Url.StartsWith("https://", StringComparison.InvariantCulture) && !x.Url.StartsWith("http://", StringComparison.InvariantCulture) && !x.Url.StartsWith("ftps://", StringComparison.InvariantCulture) && @@ -68,6 +68,7 @@ public List GetLocalHyperlinks(string root, string filePath) }) .ToList(); + // updating the links foreach (var link in links) { if (link.Url != null && !link.Url.Equals(markdown.Substring(link.UrlSpanStart, link.UrlSpanLength), StringComparison.Ordinal)) @@ -80,6 +81,12 @@ public List GetLocalHyperlinks(string root, string filePath) link.Url = markdown.Substring(link.UrlSpanStart, link.UrlSpanLength); } + if (link.Url?.StartsWith('~') == true) + { + // special reference to root. We need to expand that to the root folder. + link.Url = _workingFolder + link.Url.AsSpan(1).ToString(); + } + if (link.IsLocal) { int pos = link.Url!.IndexOf('#', StringComparison.InvariantCulture); diff --git a/src/DocAssembler/DocAssembler/Program.cs b/src/DocAssembler/DocAssembler/Program.cs index 19f5d81..e41ed8f 100644 --- a/src/DocAssembler/DocAssembler/Program.cs +++ b/src/DocAssembler/DocAssembler/Program.cs @@ -156,18 +156,21 @@ async Task AssembleDocumentationAsync( // INVENTORY InventoryAction inventory = new(currentFolder, config, fileService, logger); - ret &= await inventory.RunAsync(); + ret = await inventory.RunAsync(); - if (cleanup && Directory.Exists(outputFolderPath)) + if (ret != ReturnCode.Error) { - // CLEANUP OUTPUT - Directory.Delete(outputFolderPath, true); + if (cleanup && Directory.Exists(outputFolderPath)) + { + // CLEANUP OUTPUT + Directory.Delete(outputFolderPath, true); + } + + // ASSEMBLE + AssembleAction assemble = new(inventory.Files, fileService, logger); + ret = await assemble.RunAsync(); } - // ASSEMBLE - AssembleAction assemble = new(inventory.Files, fileService, logger); - ret &= await assemble.RunAsync(); - logger.LogInformation($"Command completed. Return value: {ret}."); return ret; From 902d3743aea26bd42e3b1f10e56865b5e383b569 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Thu, 28 Nov 2024 14:44:49 +0100 Subject: [PATCH 06/12] Fixed some issues --- src/DocAssembler/DocAssembler/Actions/AssembleAction.cs | 1 + src/DocAssembler/DocAssembler/Actions/InventoryAction.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs index b46d47e..cedb574 100644 --- a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs @@ -2,6 +2,7 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using System.Diagnostics; using System.Text; using System.Text.RegularExpressions; using DocAssembler.FileService; diff --git a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs index bfbcdd2..d072a61 100644 --- a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs @@ -124,7 +124,7 @@ private ReturnCode UpdateLinks() else { // we're calculating the link with the external file prefix, usualy a repo web link prefix. - string subpath = link.UrlFullPath.Substring(file.ContentSet!.SourceFolder.Length).TrimStart('/'); + string subpath = link.UrlFullPath.Substring(_workingFolder.Length).TrimStart('/'); link.DestinationFullUrl = file.ContentSet!.ExternalFilePrefix.TrimEnd('/') + "/" + subpath; } } From 107eea5b0af1bcd959e9833c71be48b3a297fc38 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Tue, 3 Dec 2024 14:12:27 +0100 Subject: [PATCH 07/12] Added global replace and prefix settings. Content settings can override --- .../DocAssembler/Actions/AssembleAction.cs | 17 ++++++++++++----- .../DocAssembler/Actions/InventoryAction.cs | 11 +++++++---- .../Configuration/AssembleConfiguration.cs | 19 +++++++++++++++++++ src/DocAssembler/DocAssembler/Program.cs | 2 +- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs index cedb574..3a3ad40 100644 --- a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Text; using System.Text.RegularExpressions; +using DocAssembler.Configuration; using DocAssembler.FileService; using Microsoft.Extensions.Logging; @@ -15,6 +16,7 @@ namespace DocAssembler.Actions; /// public class AssembleAction { + private readonly AssembleConfiguration _config; private readonly List _files; private readonly IFileService _fileService; private readonly ILogger _logger; @@ -22,14 +24,17 @@ public class AssembleAction /// /// Initializes a new instance of the class. /// + /// Configuration. /// List of files to process. /// File service. /// Logger. public AssembleAction( + AssembleConfiguration config, List files, IFileService fileService, ILogger logger) { + _config = config; _files = files; _fileService = fileService; _logger = logger; @@ -74,17 +79,19 @@ public Task RunAsync() string output = sb.ToString(); // if replacement patterns are defined, apply them to the content - int replacements = 0; - if (file.ContentSet?.ContentReplacements is not null) + // If content replacements are defined, we use that one, otherwise the global replacements. + int replacementCount = 0; + var replacements = file.ContentSet?.ContentReplacements ?? _config.ContentReplacements; + if (replacements is not null) { try { // apply all replacements - foreach (var replacement in file.ContentSet.ContentReplacements) + foreach (var replacement in replacements) { string r = replacement.Value ?? string.Empty; output = Regex.Replace(output, replacement.Expression, r); - replacements++; + replacementCount++; } } catch (Exception ex) @@ -96,7 +103,7 @@ public Task RunAsync() Directory.CreateDirectory(Path.GetDirectoryName(file.DestinationPath)!); _fileService.WriteAllText(file.DestinationPath, output); - _logger.LogInformation($"Copied '{file.SourcePath}' to '{file.DestinationPath}' with {updates.Count()} URL replacements and {replacements} content replacements."); + _logger.LogInformation($"Copied '{file.SourcePath}' to '{file.DestinationPath}' with {updates.Count()} URL replacements and {replacementCount} content replacements."); } else { diff --git a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs index d072a61..e12a0fa 100644 --- a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs @@ -115,7 +115,8 @@ private ReturnCode UpdateLinks() } else { - if (string.IsNullOrEmpty(file.ContentSet!.ExternalFilePrefix)) + var prefix = file.ContentSet!.ExternalFilePrefix ?? _config.ExternalFilePrefix; + if (string.IsNullOrEmpty(prefix)) { // ERROR: no solution to fix this reference _logger.LogCritical($"Error in a file reference. Link '{link.OriginalUrl}' in '{file.SourcePath}' cannot be resolved and no external file prefix was given."); @@ -125,7 +126,7 @@ private ReturnCode UpdateLinks() { // we're calculating the link with the external file prefix, usualy a repo web link prefix. string subpath = link.UrlFullPath.Substring(_workingFolder.Length).TrimStart('/'); - link.DestinationFullUrl = file.ContentSet!.ExternalFilePrefix.TrimEnd('/') + "/" + subpath; + link.DestinationFullUrl = prefix.TrimEnd('/') + "/" + subpath; } } } @@ -197,12 +198,14 @@ private ReturnCode GetAllFiles() fileData.DestinationPath = _fileService.GetFullPath(Path.Combine(destFolder, subpath)); // if replace patterns are defined, apply them to the destination path - if (content.UrlReplacements != null) + // content replacements will be used if defined, otherwise the global replacements are used. + var replacements = content.UrlReplacements ?? _config.UrlReplacements; + if (replacements != null) { try { // apply all replacements - foreach (var replacement in content.UrlReplacements) + foreach (var replacement in replacements) { string r = replacement.Value ?? string.Empty; fileData.DestinationPath = Regex.Replace(fileData.DestinationPath, replacement.Expression, r); diff --git a/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs b/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs index 195ab2d..90f7d09 100644 --- a/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs +++ b/src/DocAssembler/DocAssembler/Configuration/AssembleConfiguration.cs @@ -17,6 +17,25 @@ public sealed record AssembleConfiguration [JsonPropertyName("dest")] public string DestinationFolder { get; set; } = string.Empty; + /// + /// Gets or sets the global URL replacements. Can be overruled by settings. + /// + public List? UrlReplacements { get; set; } + + /// + /// Gets or sets the global content replacements. Can be overruled by settings. + /// + public List? ContentReplacements { get; set; } + + /// + /// Gets or sets the prefix for external files like source files. + /// This is for all references to files that are not part of the + /// selected files (mostly markdown and assets). + /// An example use is to prefix the URL with the url of the github repo. + /// This is the global setting, that can be overruled by settings. + /// + public string? ExternalFilePrefix { get; set; } + /// /// Gets or sets the content to process. /// diff --git a/src/DocAssembler/DocAssembler/Program.cs b/src/DocAssembler/DocAssembler/Program.cs index e41ed8f..6ecdf40 100644 --- a/src/DocAssembler/DocAssembler/Program.cs +++ b/src/DocAssembler/DocAssembler/Program.cs @@ -167,7 +167,7 @@ async Task AssembleDocumentationAsync( } // ASSEMBLE - AssembleAction assemble = new(inventory.Files, fileService, logger); + AssembleAction assemble = new(config, inventory.Files, fileService, logger); ret = await assemble.RunAsync(); } From 8ab11d5d7ccedf6622527d47df90088bc6515b53 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Tue, 3 Dec 2024 15:35:29 +0100 Subject: [PATCH 08/12] Added readme --- .../DocAssembler/Actions/AssembleAction.cs | 1 - .../DocAssembler/Actions/ConfigInitAction.cs | 3 +- .../DocAssembler/Configuration/Content.cs | 10 +- .../Configuration/FolderNamingStrategy.cs | 21 --- src/DocAssembler/README.md | 157 ++++++++++++++++++ 5 files changed, 163 insertions(+), 29 deletions(-) delete mode 100644 src/DocAssembler/DocAssembler/Configuration/FolderNamingStrategy.cs create mode 100644 src/DocAssembler/README.md diff --git a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs index 3a3ad40..3ad02da 100644 --- a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs @@ -2,7 +2,6 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // -using System.Diagnostics; using System.Text; using System.Text.RegularExpressions; using DocAssembler.Configuration; diff --git a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs index 207264b..98062fc 100644 --- a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs @@ -61,6 +61,7 @@ public async Task RunAsync() var config = new AssembleConfiguration { DestinationFolder = "out", + ExternalFilePrefix = "https://github.com/example/blob/main/", Content = [ new Content @@ -73,7 +74,6 @@ public async Task RunAsync() { SourceFolder = "docs", Files = { "**" }, - ExternalFilePrefix = "https://github.com/example/blob/main/", }, new Content { @@ -87,7 +87,6 @@ public async Task RunAsync() Value = "/", } ], - ExternalFilePrefix = "https://github.com/example/blob/main/", }, ], }; diff --git a/src/DocAssembler/DocAssembler/Configuration/Content.cs b/src/DocAssembler/DocAssembler/Configuration/Content.cs index f542cdc..9a52de3 100644 --- a/src/DocAssembler/DocAssembler/Configuration/Content.cs +++ b/src/DocAssembler/DocAssembler/Configuration/Content.cs @@ -35,6 +35,11 @@ public sealed record Content /// public List? Exclude { get; set; } + /// + /// Gets or sets a value indicating whether we need to do just a raw copy. + /// + public bool? RawCopy { get; set; } + /// /// Gets or sets the URL replacements. /// @@ -45,11 +50,6 @@ public sealed record Content /// public List? ContentReplacements { get; set; } - /// - /// Gets or sets a value indicating whether we need to do just a raw copy. - /// - public bool? RawCopy { get; set; } - /// /// Gets or sets the prefix for external files like source files. /// This is for all references to files that are not part of the diff --git a/src/DocAssembler/DocAssembler/Configuration/FolderNamingStrategy.cs b/src/DocAssembler/DocAssembler/Configuration/FolderNamingStrategy.cs deleted file mode 100644 index 0468b53..0000000 --- a/src/DocAssembler/DocAssembler/Configuration/FolderNamingStrategy.cs +++ /dev/null @@ -1,21 +0,0 @@ -// -// Copyright (c) DocFx Companion Tools. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. -// -namespace DocAssembler.Configuration; - -/// -/// Copy strategy. -/// -public enum FolderNamingStrategy -{ - /// - /// Use name of the source folder. - /// - SourceFolder, - - /// - /// Use name of the parent folder of the source. - /// - ParentFolder, -} diff --git a/src/DocAssembler/README.md b/src/DocAssembler/README.md new file mode 100644 index 0000000..aa6cb8f --- /dev/null +++ b/src/DocAssembler/README.md @@ -0,0 +1,157 @@ +# Documentation Assembler Tool + +This tool can be used to assemble documentation from various locations on disk and make sure all links still work. + +## Usage + +```text +DocAssembler [command] [options] + +Options: + --workingfolder The working folder. Default is the current folder. + --config (REQUIRED) The configuration file for the assembled documentation. + --outfolder Override the output folder for the assembled documentation in the config file. + --cleanup-output Cleanup the output folder before generating. NOTE: This will delete all folders and files! + -v, --verbose Show verbose messages of the process. + --version Show version information + -?, -h, --help Show help and usage information + +Commands: + init Intialize a configuration file in the current directory if it doesn't exist yet. +``` + +If normal return code of the tool is 0, but on error it returns 1. + +Return values: + 0 - successful. + 1 - some warnings, but process could be completed. + 2 - a fatal error occurred. + +## Warnings, errors and verbose + +If the tool encounters situations that might need some action, a warning is written to the output. Documentation is still assembled. If the tool encounters an error, an error message is written to the output. Documentation might not be assembled or complete. + +If you want to trace what the tool is doing, use the `-v or --verbose` flag to output all details of processing the files and folders and assembling content. + +## Overall process + +The overall process of this tool is: + +1. Content inventory - retrieve all folders and files that can be found with the configured content sets. In this stage we already calculate the new path in the configured output folder. Url replacements when configured are executed here (see [`Replacement`](#replacement) for more details). +2. If configured, delete the existing output folder. +3. Copy over all found files to the newly calculated location. Content replacements when configured are executed here. We also change links in markdown files to the new location of the referenced files, unless it's a 'raw copy'. Referenced files that are not found in the content sets are prefixed with the configured prefix. + +The basic idea is to define a content set that will be copied to the destination folder. The reason to do this, is because we now have the possibility to completely restructure the documentation, but also apply changes in the content. In a CI/CD process this can be used to assemble all documentation to prepare it for the use of the [DocFxTocGenerator](https://github.com/Ellerbach/docfx-companion-tools/blob/main/src/DocFxTocGenerator) to generate the table of content and then use tools as [DocFx](https://dotnet.github.io/docfx/) to generate a documentation website. The tool expects the content set to be validated for valid links. This can be done using the [DocLinkChecker](https://github.com/Ellerbach/docfx-companion-tools/blob/main/src/DocLinkChecker). + +## Configuration file + +A configuration file is used for settings. Command line parameters will overwrite these settings if provided. + +An initialized configuration file called `.docassembler.json` can be generated in the working directory by using the command: + +```shell +DocAssembler init +``` + +If a `.docassembler.json` file already exists in the working directory, an error is given that it will not be overwritten. The generated structure will look like this: + +```json +{ + "dest": "out", + "externalFilePrefix": "https://github.com/example/blob/main/", + "content": [ + { + "src": ".docfx", + "files": [ + "**" + ], + "rawCopy": true + }, + { + "src": "docs", + "files": [ + "**" + ] + }, + { + "src": "backend", + "dest": "services", + "files": [ + "**/docs/**" + ], + "urlReplacements": [ + { + "expression": "/[Dd]ocs/", + "value": "/" + } + ] + } + ] +} +``` + +### General settings + +In the general settings these properties can be set: + +| Property | Description | +| --------------------- | ------------------------------------------------------------ | +| `dest` (Required) | Destination sub-folder in the working folder to copy the assembled documentation to. This value can be overruled with the `--outfolder` command line argument. | +| `urlReplacements` | A global collection of [`Replacement`](#replacement) objects to use across content sets for URL paths. These replacements are applied to calculated destination paths for files in the content sets. This can be used to modify the path. The generated template removes /docs/ from paths and replaces it by a /. If a content set has `urlReplacements` configured, it overrules these global ones. More information can be found under [`Replacement`](#replacement). | +| `contentReplacements` | A global collection of [`Replacement`](#replacement) objects to use across content sets for content of files. These replacements are applied to all content of markdown files in the content sets. This can be used to modify for instance URLs or other content items. If a content set has `contentReplacements` configured, it overrules these global ones. More information can be found under [`Replacement`](#replacement). | +| `externalFilePrefix` | The global prefix to use for all referenced files in all content sets that are not part of the documentation, like source files. This prefix is used in combination with the sub-path from the working folder. If a content set has `externalFilePrefix` configured, it overrules this global one. | +| `content` (Required) | A collection of [`Content`](#content) objects to define all content sets to assemble. | + +### `Replacement` + +A replacement definition has these properties: + +| Property | Description | +| ------------ | ------------------------------------------------------------ | +| `expression` | A regular expression to find specific text. | +| `value` | The value that replaces the found text. Named matched subexpressions can be used here as well as explained below. | + +This type is used in collections for URL replacements or content replacements. They are applied one after another, starting with the first entry. The regular expression is used to find text that will be replaced by the value. The expressions are regular expression as described in [.NET Regular Expressions - .NET Microsoft Learn](https://learn.microsoft.com/en-us/dotnet/standard/base-types/regular-expressions). Examples can be found there as well. There are websites like [regex101: build, test, and debug regex](https://regex101.com/) to build, debug and validate the expression you need. + +#### Using named matched subexpressions + +Sometimes you want to find specific content, but also reuse parts of it in the value replacement. An example would be to find all `AB#1234` notations and replace it by a URL to the referenced Azure Boards work-item or GitHub item. But in this case we want to use the ID (1234) in the value. To do that, you can use [Named matched subexpressions](https://learn.microsoft.com/en-us/dotnet/standard/base-types/grouping-constructs-in-regular-expressions#named-matched-subexpressions). + +This expression could be used to find all those references: + +```regex +(?
[$\\s])AB#(?[0-9]{3,6})
+```
+
+As we don't want to find a link like `[AB#1234](https://...)`, we look for all AB# references that are at the start of a line (using the `$` tag) or are prefixed by a whitespace (using the `\s` tag). As we need to keep that prefix in place, we capture it as a named subexpression called `pre`.
+
+> [!NOTE]
+>
+> As the expression is configured in a string in a JSON file, special characters like back-slashes need to be escaped by an (extra) back-slash.
+
+The second part is to get the numbers after the AB# text. This is configured here to be between 3 and 6 characters. We also want to reuse this ID in the value, so we capture it as a named subexpression called `id`.
+
+In the value we can reuse these named subexpression like this:
+
+```text
+${pre}[AB#${id}](https://dev.azure.com/[your organization]/_workitems/edit/${id})
+```
+
+We start with the `pre` value, after which we build a markdown link with AB# combined with the `id` as the text and the `id` as parameter for the URL. We reference an Azure Board work item here. Of course you need to replace the `[your organization]` with the proper value for your ADO environment here.
+
+### `Content`
+
+The content is defined with these properties:
+
+| Property         | Description                                                  |
+| ---------------- | ------------------------------------------------------------ |
+| `src` (Required) | The source sub-folder relative to the working folder.        |
+| `dest`           | An optional destination sub-folder path in the output folder. If this is not given, the relative path to the source folder is used. |
+| `files`          | This is a  [File Globbing in .NET](https://learn.microsoft.com/en-us/dotnet/core/extensions/file-globbing) pattern. Make sure to also include all needed files for documentation like images and assets. |
+| `exclude`        | This is a  [File Globbing in .NET](https://learn.microsoft.com/en-us/dotnet/core/extensions/file-globbing) pattern. This can be used to exclude specific folders or files from the content set. |
+| `rawCopy` | If this value is `true` then we don't look for any links in markdown files and therefor also don't fix them. This can be used for raw content you want to include in the documentation set like `.docfx.json`, templates and such. |
+| `urlReplacements`     | A collection of [`Replacement`](#replacement) objects to use for URL paths in this content set, overruling any global setting. These replacements are applied to calculated destination paths for files in the content sets. This can be used to modify the path. The generated template removes /docs/ from paths and replaces it by a /. More information can be found under [`Replacement`](#replacement). |
+| `contentReplacements` | A collection of [`Replacement`](#replacement) objects to use for content of files in this content set, overruling any global setting. These replacements are applied to all content of markdown files in the content sets. This can be used to modify for instance URLs or other content items. More information can be found under [`Replacement`](#replacement). |
+| `externalFilePrefix`  | The prefix to use for all referenced files in this content sets that are not part of the complete documentation set, like source files. It overrides any global prefix. This prefix is used in combination with the sub-path from the working folder. |
+
+

From f57508b8adbc1524ac9baa41cab97bad4dcb5bf5 Mon Sep 17 00:00:00 2001
From: Martin Tirion 
Date: Thu, 12 Dec 2024 19:59:07 +0100
Subject: [PATCH 09/12] Added tests

---
 .../DocAssembler.Test/AssembleActionTests.cs  | 348 ++++++++++++++++
 .../ConfigInitActionTests.cs                  |  72 ++++
 .../DocAssembler.Test/Directory.Build.props   |   3 +
 .../DocAssembler.Test.csproj                  |  27 ++
 .../DocAssembler.Test/FileInfoServiceTests.cs |  70 ++++
 .../DocAssembler.Test/FileServiceTests.cs     |  56 +++
 .../Helpers/MarkdownExtensions.cs             | 130 ++++++
 .../Helpers/MockFileService.cs                | 386 ++++++++++++++++++
 .../DocAssembler.Test/Helpers/MockLogger.cs   | 100 +++++
 .../DocAssembler.Test/InventoryActionTests.cs | 307 ++++++++++++++
 src/DocAssembler/DocAssembler.sln             |   8 +-
 .../DocAssembler/Actions/AssembleAction.cs    |   3 +-
 .../DocAssembler/Actions/ConfigInitAction.cs  |  11 +-
 .../DocAssembler/Actions/InventoryAction.cs   |   7 +-
 .../DocAssembler/Configuration/Content.cs     |   1 +
 .../DocAssembler/DocAssembler.csproj          |   2 +-
 .../FileService/FileInfoService.cs            |   8 +-
 .../DocAssembler/FileService/MarkdownLink.cs  |  16 -
 src/DocAssembler/DocAssembler/README.md       | 355 ----------------
 .../DocAssembler/Utils/SerializationUtil.cs   |   2 +-
 20 files changed, 1526 insertions(+), 386 deletions(-)
 create mode 100644 src/DocAssembler/DocAssembler.Test/AssembleActionTests.cs
 create mode 100644 src/DocAssembler/DocAssembler.Test/ConfigInitActionTests.cs
 create mode 100644 src/DocAssembler/DocAssembler.Test/Directory.Build.props
 create mode 100644 src/DocAssembler/DocAssembler.Test/DocAssembler.Test.csproj
 create mode 100644 src/DocAssembler/DocAssembler.Test/FileInfoServiceTests.cs
 create mode 100644 src/DocAssembler/DocAssembler.Test/FileServiceTests.cs
 create mode 100644 src/DocAssembler/DocAssembler.Test/Helpers/MarkdownExtensions.cs
 create mode 100644 src/DocAssembler/DocAssembler.Test/Helpers/MockFileService.cs
 create mode 100644 src/DocAssembler/DocAssembler.Test/Helpers/MockLogger.cs
 create mode 100644 src/DocAssembler/DocAssembler.Test/InventoryActionTests.cs
 delete mode 100644 src/DocAssembler/DocAssembler/FileService/MarkdownLink.cs
 delete mode 100644 src/DocAssembler/DocAssembler/README.md

diff --git a/src/DocAssembler/DocAssembler.Test/AssembleActionTests.cs b/src/DocAssembler/DocAssembler.Test/AssembleActionTests.cs
new file mode 100644
index 0000000..65336e4
--- /dev/null
+++ b/src/DocAssembler/DocAssembler.Test/AssembleActionTests.cs
@@ -0,0 +1,348 @@
+// 
+// Copyright (c) DocFx Companion Tools. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+// 
+using Bogus;
+using DocAssembler.Actions;
+using DocAssembler.Configuration;
+using DocAssembler.Test.Helpers;
+using FluentAssertions;
+using Microsoft.Extensions.Logging;
+
+namespace DocAssembler.Test;
+
+public class AssembleActionTests
+{
+    private Faker _faker = new();
+    private MockFileService _fileService = new();
+    private MockLogger _mockLogger = new();
+    private ILogger _logger;
+
+    private string _workingFolder = string.Empty;
+    private string _outputFolder = string.Empty;
+
+    public AssembleActionTests()
+    {
+        _fileService.FillDemoSet();
+        _logger = _mockLogger.Logger;
+
+        _workingFolder = _fileService.Root;
+        _outputFolder = Path.Combine(_fileService.Root, "out");
+    }
+
+    [Fact]
+    public async void Run_MinimumConfigAllCopied()
+    {
+        // arrange
+        AssembleConfiguration config = new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            Content =
+            [
+                new Content
+                    {
+                        SourceFolder = ".docfx",
+                        Files = { "**" },
+                        RawCopy = true,         // just copy the content
+                    }
+                ],
+        };
+
+        // all files in .docfx and docs-children
+        int count = _fileService.Files.Count;
+        var expected = _fileService.Files.Where(x => x.Key.Contains("/.docfx/"));
+
+        InventoryAction inventory = new(_workingFolder, config, _fileService, _logger);
+        await inventory.RunAsync();
+        AssembleAction action = new(config, inventory.Files, _fileService, _logger);
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        // expect files to be original count + expected files and folders + new "out" folder.
+        _fileService.Files.Should().HaveCount(count + expected.Count() + 1);
+
+        // validate file content is copied
+        var file = expected.Single(x => x.Key.EndsWith("index.md"));
+        var expectedPath = file.Key.Replace("/.docfx/", $"/{config.DestinationFolder}/");
+        var newFile = _fileService.Files.SingleOrDefault(x => x.Key == expectedPath);
+        newFile.Should().NotBeNull();
+        newFile.Value.Should().Be(file.Value);
+    }
+
+    [Fact]
+    public async void Run_MinimumConfigAllCopied_WithGlobalContentReplace()
+    {
+        // arrange
+        AssembleConfiguration config = new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            ContentReplacements =
+            [
+                new Replacement
+                {
+                    Expression = @"(?
[$\s])AB#(?[0-9]{3,6})",
+                    Value = @"${pre}[AB#${id}](https://dev.azure.com/MyCompany/MyProject/_workitems/edit/${id})"
+                }
+            ],
+            Content =
+            [
+                new Content
+                    {
+                        SourceFolder = "docs",
+                        DestinationFolder = "general",
+                        Files = { "**" },
+                    }
+                ],
+        };
+
+        // all files in .docfx and docs-children
+        int count = _fileService.Files.Count;
+        var expected = _fileService.Files.Where(x => x.Key.StartsWith($"{_fileService.Root}/docs/"));
+
+        InventoryAction inventory = new(_workingFolder, config, _fileService, _logger);
+        await inventory.RunAsync();
+        AssembleAction action = new(config, inventory.Files, _fileService, _logger);
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        // expect files to be original count + expected files and folders + new "out" folder.
+        _fileService.Files.Should().HaveCount(count + expected.Count() + 1);
+
+        // validate file content is copied with changed AB# reference
+        var file = expected.Single(x => x.Key == $"{_fileService.Root}/docs/guidelines/documentation-guidelines.md");
+        var expectedPath = file.Key.Replace("/docs/", $"/{config.DestinationFolder}/general/");
+        var newFile = _fileService.Files.SingleOrDefault(x => x.Key == expectedPath);
+        newFile.Should().NotBeNull();
+        string[] lines = _fileService.ReadAllLines(newFile.Key);
+        var testLine = lines.Single(x => x.StartsWith("STANDARD:"));
+        testLine.Should().NotBeNull();
+        testLine.Should().Be("STANDARD: [AB#1234](https://dev.azure.com/MyCompany/MyProject/_workitems/edit/1234) reference");
+    }
+
+    [Fact]
+    public async void Run_MinimumConfigAllCopied_WithContentReplace()
+    {
+        // arrange
+        AssembleConfiguration config = new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            Content =
+            [
+                new Content
+                    {
+                        SourceFolder = "docs",
+                        DestinationFolder = "general",
+                        Files = { "**" },
+                        ContentReplacements =
+                        [
+                            new Replacement
+                            {
+                                Expression = @"(?
[$\s])AB#(?[0-9]{3,6})",
+                                Value = @"${pre}[AB#${id}](https://dev.azure.com/MyCompany/MyProject/_workitems/edit/${id})"
+                            }
+                        ],
+                    }
+                ],
+        };
+
+        // all files in .docfx and docs-children
+        int count = _fileService.Files.Count;
+        var expected = _fileService.Files.Where(x => x.Key.StartsWith($"{_fileService.Root}/docs/"));
+
+        InventoryAction inventory = new(_workingFolder, config, _fileService, _logger);
+        await inventory.RunAsync();
+        AssembleAction action = new(config, inventory.Files, _fileService, _logger);
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        // expect files to be original count + expected files and folders + new "out" folder.
+        _fileService.Files.Should().HaveCount(count + expected.Count() + 1);
+
+        // validate file content is copied with changed AB# reference
+        var file = expected.Single(x => x.Key == $"{_fileService.Root}/docs/guidelines/documentation-guidelines.md");
+        var expectedPath = file.Key.Replace("/docs/", $"/{config.DestinationFolder}/general/");
+        var newFile = _fileService.Files.SingleOrDefault(x => x.Key == expectedPath);
+        newFile.Should().NotBeNull();
+        string[] lines = _fileService.ReadAllLines(newFile.Key);
+        var testLine = lines.Single(x => x.StartsWith("STANDARD:"));
+        testLine.Should().NotBeNull();
+        testLine.Should().Be("STANDARD: [AB#1234](https://dev.azure.com/MyCompany/MyProject/_workitems/edit/1234) reference");
+    }
+
+    [Fact]
+    public async void Run_MinimumConfigAllCopied_ContentShouldOverrideGlobal()
+    {
+        // arrange
+        AssembleConfiguration config = new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            ContentReplacements =
+            [
+                new Replacement
+                {
+                    Expression = @"(?
[$\s])AB#(?[0-9]{3,6})",
+                    Value = @"${pre}[AB#${id}](https://dev.azure.com/MyCompany/MyProject/_workitems/edit/${id})"
+                }
+            ],
+            Content =
+            [
+                new Content
+                    {
+                        SourceFolder = "docs",
+                        DestinationFolder = "general",
+                        Files = { "**" },
+                        ContentReplacements = [],
+                    }
+                ],
+        };
+
+        // all files in .docfx and docs-children
+        int count = _fileService.Files.Count;
+        var expected = _fileService.Files.Where(x => x.Key.StartsWith($"{_fileService.Root}/docs/"));
+
+        InventoryAction inventory = new(_workingFolder, config, _fileService, _logger);
+        await inventory.RunAsync();
+        AssembleAction action = new(config, inventory.Files, _fileService, _logger);
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        // expect files to be original count + expected files and folders + new "out" folder.
+        _fileService.Files.Should().HaveCount(count + expected.Count() + 1);
+
+        // validate file content is copied with changed AB# reference
+        var file = expected.Single(x => x.Key == $"{_fileService.Root}/docs/guidelines/documentation-guidelines.md");
+        var expectedPath = file.Key.Replace("/docs/", $"/{config.DestinationFolder}/general/");
+        var newFile = _fileService.Files.SingleOrDefault(x => x.Key == expectedPath);
+        newFile.Should().NotBeNull();
+        string[] lines = _fileService.ReadAllLines(newFile.Key);
+        var testLine = lines.Single(x => x.StartsWith("STANDARD:"));
+        testLine.Should().NotBeNull();
+        testLine.Should().Be("STANDARD: AB#1234 reference");
+    }
+
+    [Fact]
+    public async void Run_StandardConfigAllCopied()
+    {
+        // arrange
+        AssembleConfiguration config = GetStandardConfiguration();
+
+        // all files in .docfx and docs-children
+        int count = _fileService.Files.Count;
+        var expected = _fileService.Files.Where(x => x.Key.Contains("/.docfx/") ||
+                                                     x.Key.Contains("/docs/"));
+
+        InventoryAction inventory = new(_workingFolder, config, _fileService, _logger);
+        await inventory.RunAsync();
+        AssembleAction action = new(config, inventory.Files, _fileService, _logger);
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        // expect files to be original count + expected files and folders + new "out" folder.
+        _fileService.Files.Should().HaveCount(count + expected.Count() + 1);
+
+        // validate file content is copied with changed AB# reference
+        var file = expected.Single(x => x.Key == $"{_fileService.Root}/docs/getting-started/README.md");
+        var expectedPath = file.Key.Replace("/docs/", $"/{config.DestinationFolder}/general/");
+        var newFile = _fileService.Files.SingleOrDefault(x => x.Key == expectedPath);
+        newFile.Should().NotBeNull();
+
+        string[] lines = _fileService.ReadAllLines(newFile.Key);
+
+        string testLine = lines.Single(x => x.StartsWith("EXTERNAL:"));
+        testLine.Should().Be("EXTERNAL: [.docassemble.json](https://github.com/example/blob/main/.docassemble.json)");
+
+        testLine = lines.Single(x => x.StartsWith("RESOURCE:"));
+        testLine.Should().Be("RESOURCE: ![computer](assets/computer.jpg)");
+
+        testLine = lines.Single(x => x.StartsWith("PARENT-DOC:"));
+        testLine.Should().Be("PARENT-DOC: [Docs readme](../README.md)");
+
+        testLine = lines.Single(x => x.StartsWith("RELATIVE-DOC:"));
+        testLine.Should().Be("RELATIVE-DOC: [Documentation guidelines](../guidelines/documentation-guidelines.md)");
+
+        testLine = lines.Single(x => x.StartsWith("ANOTHER-DOCS-TREE:"));
+        testLine.Should().Be("ANOTHER-DOCS-TREE: [System Copilot](../tools/system-copilot/README.md#usage)");
+
+        testLine = lines.Single(x => x.StartsWith("ANOTHER-DOCS-TREE-BACKSLASH:"));
+        testLine.Should().Be("ANOTHER-DOCS-TREE-BACKSLASH: [System Copilot](../tools/system-copilot/README.md#usage)");
+    }
+
+    private AssembleConfiguration GetStandardConfiguration()
+    {
+        return new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            ExternalFilePrefix = "https://github.com/example/blob/main/",
+            UrlReplacements =
+            [
+                new Replacement
+                {
+                    Expression = @"/[Dd]ocs/",
+                    Value = "/"
+                }
+            ],
+            ContentReplacements =
+            [
+                new Replacement
+                {
+                    Expression = @"(?
[$\s])AB#(?[0-9]{3,6})",
+                    Value = @"${pre}[AB#${id}](https://dev.azure.com/MyCompany/MyProject/_workitems/edit/${id})"
+                },
+                new Replacement     // Remove markdown style table of content
+                {
+                    Expression = @"\[\[_TOC_\]\]",
+                    Value = ""
+                }
+            ],
+            Content =
+            [
+                new Content
+                    {
+                        SourceFolder = ".docfx",
+                        Files = { "**" },
+                        RawCopy = true,         // just copy the content
+                        UrlReplacements = []    // reset URL replacements
+                    },
+                    new Content
+                    {
+                        SourceFolder = "docs",
+                        DestinationFolder = "general",
+                        Files = { "**" },
+                    },
+                    new Content
+                    {
+                        SourceFolder = "shared",    // part of general docs
+                        DestinationFolder = "general/shared",
+                        Files = { "**/docs/**" },
+                    },
+                    new Content
+                    {
+                        SourceFolder = "tools",     // part of general docs
+                        DestinationFolder = "general/tools",
+                        Files = { "**/docs/**" },
+                    },
+                    new Content
+                    {
+                        SourceFolder = "backend",
+                        DestinationFolder = "services", // change name to services
+                        Files = { "**/docs/**" },
+                    },
+                ],
+        };
+    }
+}
diff --git a/src/DocAssembler/DocAssembler.Test/ConfigInitActionTests.cs b/src/DocAssembler/DocAssembler.Test/ConfigInitActionTests.cs
new file mode 100644
index 0000000..cd707b5
--- /dev/null
+++ b/src/DocAssembler/DocAssembler.Test/ConfigInitActionTests.cs
@@ -0,0 +1,72 @@
+// 
+// Copyright (c) DocFx Companion Tools. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+// 
+using Bogus;
+using DocAssembler.Actions;
+using DocAssembler.Configuration;
+using DocAssembler.Test.Helpers;
+using DocAssembler.Utils;
+using FluentAssertions;
+using Microsoft.Extensions.Logging;
+
+namespace DocAssembler.Test;
+
+public class ConfigInitActionTests
+{
+    private Faker _faker = new();
+    private MockFileService _fileService = new();
+    private MockLogger _mockLogger = new();
+    private ILogger _logger;
+    private string _outputFolder = string.Empty;
+
+    public ConfigInitActionTests()
+    {
+        _fileService.FillDemoSet();
+        _logger = _mockLogger.Logger;
+        _outputFolder = Path.Combine(_fileService.Root, "out");
+    }
+
+    [Fact]
+    public async void Run_ConfigShouldBeCreated()
+    {
+        // arrange
+        ConfigInitAction action = new(_outputFolder, _fileService, _logger);
+        int count = _fileService.Files.Count;
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        _fileService.Files.Count.Should().Be(count + 1);
+
+        // read generated content and see if it deserializes
+        string content = _fileService.ReadAllText(_fileService.Files.Last().Key);
+        var config = SerializationUtil.Deserialize(content);
+        config.Should().NotBeNull();
+        config.DestinationFolder.Should().Be("out");
+    }
+
+    [Fact]
+    public async void Run_ConfigShouldNotBeCreatedWhenExists()
+    {
+        // arrange
+        ConfigInitAction action = new(_outputFolder, _fileService, _logger);
+        var folder = _fileService.AddFolder(_outputFolder);
+        var fileContents = "some content";
+        _fileService.AddFile(folder, ".docassembler.json", fileContents);
+        int count = _fileService.Files.Count;
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Error);
+        _fileService.Files.Count.Should().Be(count); // nothing added
+
+        // read file to see if still has the same content
+        string content = _fileService.ReadAllText(_fileService.Files.Last().Key);
+        content.Should().Be(fileContents);
+    }
+}
diff --git a/src/DocAssembler/DocAssembler.Test/Directory.Build.props b/src/DocAssembler/DocAssembler.Test/Directory.Build.props
new file mode 100644
index 0000000..63f85d6
--- /dev/null
+++ b/src/DocAssembler/DocAssembler.Test/Directory.Build.props
@@ -0,0 +1,3 @@
+
+  
+
diff --git a/src/DocAssembler/DocAssembler.Test/DocAssembler.Test.csproj b/src/DocAssembler/DocAssembler.Test/DocAssembler.Test.csproj
new file mode 100644
index 0000000..faf2b6a
--- /dev/null
+++ b/src/DocAssembler/DocAssembler.Test/DocAssembler.Test.csproj
@@ -0,0 +1,27 @@
+
+
+  
+    net8.0
+    enable
+    enable
+
+    false
+    true
+  
+
+  
+    
+    
+    
+    
+  
+
+  
+    
+  
+
+  
+    
+  
+
+
diff --git a/src/DocAssembler/DocAssembler.Test/FileInfoServiceTests.cs b/src/DocAssembler/DocAssembler.Test/FileInfoServiceTests.cs
new file mode 100644
index 0000000..ce6cc4b
--- /dev/null
+++ b/src/DocAssembler/DocAssembler.Test/FileInfoServiceTests.cs
@@ -0,0 +1,70 @@
+// 
+// Copyright (c) DocFx Companion Tools. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+// 
+using Bogus;
+using DocAssembler.FileService;
+using DocAssembler.Test.Helpers;
+using FluentAssertions;
+using Microsoft.Extensions.Logging;
+
+namespace DocAssembler.Test;
+
+public class FileInfoServiceTests
+{
+    private Faker _faker = new();
+    private MockFileService _fileService = new();
+    private MockLogger _mockLogger = new();
+    private ILogger _logger;
+
+    public FileInfoServiceTests()
+    {
+        _fileService.FillDemoSet();
+        _logger = _mockLogger.Logger;
+    }
+
+    [Fact]
+    public void GetLocalHyperlinks_GetAllWithoutResourceOrWeblink()
+    {
+        // arrange
+        FileInfoService service = new(_fileService.Root, _fileService, _logger);
+
+        // act
+        var links = service.GetLocalHyperlinks("docs", "docs/getting-started/README.md");
+
+        // assert
+        links.Should().NotBeNull();
+        links.Should().HaveCount(7);
+
+        // testing a correction in our code, as the original is parsed weird by Markdig.
+        // reason is that back-slashes in links are formally not supported.
+        links[6].OriginalUrl.Should().Be(@"..\..\tools\system-copilot\docs\README.md#usage");
+        links[6].Url.Should().Be(@"..\..\tools\system-copilot\docs\README.md#usage");
+        links[6].UrlWithoutTopic.Should().Be(@"..\..\tools\system-copilot\docs\README.md");
+        links[6].UrlTopic.Should().Be("#usage");
+    }
+
+    [Fact]
+    public void GetLocalHyperlinks_SkipEmptyLink()
+    {
+        // arrange
+        FileInfoService service = new(_fileService.Root, _fileService, _logger);
+
+        // act
+        var links = service.GetLocalHyperlinks("docs", "docs/guidelines/documentation-guidelines.md");
+
+        // assert
+        links.Should().NotBeNull();
+        links.Should().BeEmpty();
+    }
+
+    [Fact]
+    public void GetLocalHyperlinks_NotExistingFileThrows()
+    {
+        // arrange
+        FileInfoService service = new(_fileService.Root, _fileService, _logger);
+
+        // act
+        Assert.Throws(() => _ = service.GetLocalHyperlinks("docs", "docs/not-existing/phantom-file.md"));
+    }
+}
diff --git a/src/DocAssembler/DocAssembler.Test/FileServiceTests.cs b/src/DocAssembler/DocAssembler.Test/FileServiceTests.cs
new file mode 100644
index 0000000..2cd073c
--- /dev/null
+++ b/src/DocAssembler/DocAssembler.Test/FileServiceTests.cs
@@ -0,0 +1,56 @@
+// 
+// Copyright (c) DocFx Companion Tools. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+// 
+using System.Text.RegularExpressions;
+using Bogus;
+using DocAssembler.Test.Helpers;
+using FluentAssertions;
+using Microsoft.Extensions.Logging;
+
+namespace DocAssembler.Test;
+
+public class FileServiceTests
+{
+    private Faker _faker = new();
+    private MockFileService _fileService = new();
+    private MockLogger _mockLogger = new();
+    private ILogger _logger;
+
+    private string _workingFolder = string.Empty;
+    private string _outputFolder = string.Empty;
+
+    public FileServiceTests()
+    {
+        _fileService.FillDemoSet();
+        _logger = _mockLogger.Logger;
+
+        _workingFolder = _fileService.Root;
+        _outputFolder = Path.Combine(_fileService.Root, "out");
+    }
+
+    [InlineData("/Git/Project/shared", "/Git/Project/shared/dotnet/MyLibrary/docs/README.md", "**/docs/**", true)]
+    [InlineData("/Git/Project/shared", "/Git/Project/shared/dotnet/MyLibrary/src/README.md", "**/docs/**", false)]
+    [InlineData("/Git/Project/shared", "/Git/Project/shared/dotnet/MyLibrary/docs/README.md", "**/*.md", true)]
+    [InlineData("/Git/Projec/sharedt", "/Git/Project/shared/dotnet/MyLibrary/docs/images/machine.jpg", "**/*.md", false)]
+    [InlineData("/Git/Project/shared", "/Git/Project/shared/dotnet/MyLibrary/docs/README.md", "**", true)]
+    [InlineData("/Git/Project/shared", "/Git/Project/README.md", "**", false)]
+    [InlineData("/Git/Project/shared", "/Git/Project/shared/dotnet/MyLibrary/src/MyProject.Test.csproj", "**/*.Test.*", true)]
+    [InlineData("/Git/Project/shared", "/Git/Project/shared/dotnet/MyLibrary/src/MyProject.Test.csproj", "**/*Test*", true)]
+    [InlineData("/Git/Project/shared", "/Git/Project/shared/README.md", "*.md", true)]
+    [InlineData("/Git/Project/shared", "/Git/Project/toc.yml", "*.md", false)]
+    [InlineData("/Git/Project/shared", "/Git/Project/shared/dotnet/MyLibrary/docs/README.md", "*", false)]
+    [InlineData("/Git/Project/shared", "/Git/Project/shared/README.md", "*", true)]
+    [InlineData("/Git/Project/shared", "/Git/Project/shared/dotnet/MyLibrary/src/MyProject.Tests.csproj", @"**/*\.Test\.*", false)]
+    [InlineData("/Git/Project/backend", "/Git/Project/backend/docs/README.md", "**/docs/**", true)]
+    [Theory]
+    public void GlobToRegex(string root, string input, string pattern, bool selected)
+    {
+        // test of the Glob to Regex method in MockFileService class
+        // to make sure we're having the right pattern to match files for the tests.
+        string regex = _fileService.GlobToRegex(root, pattern);
+        var ret = Regex.Match(input, regex).Success;
+        ret.Should().Be(selected);
+    }
+}
+
diff --git a/src/DocAssembler/DocAssembler.Test/Helpers/MarkdownExtensions.cs b/src/DocAssembler/DocAssembler.Test/Helpers/MarkdownExtensions.cs
new file mode 100644
index 0000000..44ca30f
--- /dev/null
+++ b/src/DocAssembler/DocAssembler.Test/Helpers/MarkdownExtensions.cs
@@ -0,0 +1,130 @@
+// 
+// Copyright (c) DocFx Companion Tools. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+// 
+using Bogus;
+
+namespace DocAssembler.Test.Helpers;
+
+internal static class MarkdownExtensions
+{
+    internal static string AddHeading(this string s, string title, int level)
+    {
+        var content = $"{new string('#', level)} {title}" + Environment.NewLine + Environment.NewLine;
+        if (string.IsNullOrEmpty(s))
+        {
+            return content;
+        }
+        return s + Environment.NewLine + content;
+    }
+
+    internal static string AddParagraphs(this string s, int count = 1)
+    {
+        var faker = new Faker();
+        var content = (count == 1 ? faker.Lorem.Paragraph() : faker.Lorem.Paragraphs(count)) + Environment.NewLine;
+        if (string.IsNullOrEmpty(s))
+        {
+            return content;
+        }
+        return s + Environment.NewLine + content;
+    }
+
+    internal static string AddResourceLink(this string s, string url)
+    {
+        var faker = new Faker();
+        var content = $" ![some resource {faker.Random.Int(1)}]({url})" + Environment.NewLine;
+        if (string.IsNullOrEmpty(s))
+        {
+            return content;
+        }
+        return s + Environment.NewLine + content;
+    }
+
+    internal static string AddLink(this string s, string url)
+    {
+        var faker = new Faker();
+        var content = $" [some link {faker.Random.Int(1)}]({url})" + Environment.NewLine;
+        if (string.IsNullOrEmpty(s))
+        {
+            return content;
+        }
+        return s + Environment.NewLine + content;
+    }
+
+    internal static string AddCodeLink(this string s, string name, string url)
+    {
+        var faker = new Faker();
+        var content = $" [!code-csharp[{name}]({url})]" + Environment.NewLine;
+        if (string.IsNullOrEmpty(s))
+        {
+            return content;
+        }
+        return s + Environment.NewLine + content;
+    }
+
+    internal static string AddRawLink(this string s, string name, string url)
+    {
+        var faker = new Faker();
+        var content = $" [{name}]({url})" + Environment.NewLine;
+        if (string.IsNullOrEmpty(s))
+        {
+            return content;
+        }
+        return s + Environment.NewLine + content;
+    }
+
+    internal static string AddTableStart(this string s, int columns = 3)
+    {
+        var faker = new Faker();
+        var content = "|";
+        for (var col = 0; col < columns; col++)
+        {
+            content += $" {faker.Lorem.Words(2)} |";
+        }
+        content += Environment.NewLine;
+        for (var col = 0; col < columns; col++)
+        {
+            content += $" --- |";
+        }
+        content += Environment.NewLine;
+        if (string.IsNullOrEmpty(s))
+        {
+            return content;
+        }
+        return s + Environment.NewLine + content;
+    }
+
+    internal static string AddTableRow(this string s, params string[] columns)
+    {
+        var faker = new Faker();
+        var content = "|";
+        foreach (var col in columns)
+        {
+            content += $" {col} |";
+        }
+        content += Environment.NewLine;
+        if (string.IsNullOrEmpty(s))
+        {
+            return content;
+        }
+        return s + Environment.NewLine + content;
+    }
+
+    internal static string AddNewLine(this string s)
+    {
+        if (string.IsNullOrEmpty(s))
+        {
+            return Environment.NewLine;
+        }
+        return s + Environment.NewLine;
+    }
+
+    internal static string AddRaw(this string s, string markdown)
+    {
+        if (string.IsNullOrEmpty(s))
+        {
+            return markdown;
+        }
+        return s + markdown + Environment.NewLine;
+    }
+}
diff --git a/src/DocAssembler/DocAssembler.Test/Helpers/MockFileService.cs b/src/DocAssembler/DocAssembler.Test/Helpers/MockFileService.cs
new file mode 100644
index 0000000..55a85f4
--- /dev/null
+++ b/src/DocAssembler/DocAssembler.Test/Helpers/MockFileService.cs
@@ -0,0 +1,386 @@
+// 
+// Copyright (c) DocFx Companion Tools. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+// 
+using System.Text;
+using System.Text.RegularExpressions;
+using DocAssembler.FileService;
+
+namespace DocAssembler.Test.Helpers;
+
+public class MockFileService : IFileService
+{
+    public string Root;
+
+    public Dictionary Files { get; set; } = new();
+
+    public MockFileService()
+    {
+        // determine if we're testing on Windows. If not, use linux paths.
+        if (Path.IsPathRooted("c://"))
+        {
+            // windows
+            Root = "c:/Git/Project";
+        }
+        else
+        {
+            // linux
+            Root = "/Git/Project";
+        }
+    }
+
+    public void FillDemoSet()
+    {
+        Files.Clear();
+
+        // make sure that root folders are available
+        EnsurePath(Root);
+        
+        // 4 files
+        var folder = AddFolder(".docfx");
+        AddFile(folder, "docfx.json", "");
+        AddFile(folder, "index.md", string.Empty
+            .AddHeading("Test Repo", 1)
+            .AddParagraphs(1)
+            .AddRawLink("Keyboard", "images/keyboard.jpg")
+            .AddParagraphs(1)
+            .AddRawLink("setup your dev machine", "./general/getting-started/setup-dev-machine.md"));
+        AddFile(folder, "toc.yml",
+@"---
+- name: General
+  href: general/README.md
+- name: Services
+  href: services/README.md");
+        folder = AddFolder(".docfx/images");
+        AddFile(folder, "keyboard.jpg", "");
+
+        // docs: 1 + 2 + 1 + 3 = 7 files
+        folder = AddFolder($"docs");
+        AddFile(folder, "README.md", string.Empty
+            .AddHeading("Documentation Readme", 1)
+            .AddParagraphs(3));
+        folder = AddFolder("docs/getting-started");
+        AddFile(folder, "README.md", string.Empty
+            .AddHeading("Getting Started", 1)
+            .AddRaw("EXTERNAL: [.docassemble.json](../../.docassemble.json)")
+            .AddRaw("WEBLINK: [Microsoft](https://www.microsoft.com)")
+            .AddRaw("RESOURCE: ![computer](assets/computer.jpg)")
+            .AddRaw("PARENT-DOC: [Docs readme](../README.md)")
+            .AddRaw("RELATIVE-DOC: [Documentation guidelines](../guidelines/documentation-guidelines.md)")
+            .AddRaw("ANOTHER-SUBFOLDER-DOC: [Documentation guidelines](../guidelines/documentation-guidelines.md)")
+            .AddRaw("ANOTHER-DOCS-TREE: [System Copilot](../../tools/system-copilot/docs/README.md#usage)")
+            .AddRaw("ANOTHER-DOCS-TREE-BACKSLASH: [System Copilot](..\\..\\tools\\system-copilot\\docs\\README.md#usage)"));
+        folder = AddFolder("docs/getting-started/assets");
+        AddFile(folder, "computer.jpg", "");
+        folder = AddFolder("docs/guidelines");
+        AddFile(folder, "documentation-guidelines.md", string.Empty
+            .AddHeading("Documentation Guidelines", 1)
+            .AddRaw("STANDARD: AB#1234 reference")
+            .AddRaw("AB#4321 at START")
+            .AddRaw("EMPTY-LINK: [AB#1000]() is okay."));
+        AddFile(folder, "dotnet-guidelines.md", string.Empty
+            .AddHeading(".NET Guidelines", 1)
+            .AddParagraphs(3));
+
+        // 
+        folder = AddFolder("backend");
+        folder = AddFolder("backend/docs");
+        AddFile(folder, "README.md", string.Empty
+            .AddHeading("Backend", 1)
+            .AddParagraphs(2));
+        folder = AddFolder("backend/app1");
+        folder = AddFolder("backend/app1/docs");
+        AddFile(folder, "README.md", string.Empty
+            .AddHeading("App1", 1)
+            .AddRaw(@"We're using the [system copilot](../../../tools/system-copilot/docs/README.md#usage)")
+            .AddParagraphs(2)
+            );
+        folder = AddFolder("backend/app1/src");
+        AddFile(folder, "app1.cs", "");
+        folder = AddFolder("backend/subsystem1");
+        folder = AddFolder("backend/subsystem1/docs");
+        AddFile(folder, "explain-subsystem.md", string.Empty
+            .AddHeading("Subsystem 1", 1)
+            .AddParagraphs(3));
+        folder = AddFolder("backend/subsystem1/app20");
+        folder = AddFolder("backend/subsystem1/app20/docs");
+        AddFile(folder, "README.md", string.Empty
+            .AddHeading("app20", 1)
+            .AddRaw(
+@"This is part of [the subsystem1](../../docs/explain-subsystem1.md).
+
+We're following the [Documentation Guidelines](../../../../docs/guidelines/documentation-guidelines.md)")
+            .AddParagraphs(1)
+            .AddRaw("It's also important to look at [the code](../src/app20.cs)")
+            .AddParagraphs(3));
+        folder = AddFolder("backend/subsystem1/app20/src");
+        AddFile(folder, "app20.cs", "");
+        folder = AddFolder("backend/subsystem1/app30");
+        folder = AddFolder("backend/subsystem1/app30/docs");
+        AddFile(folder, "README.md", string.Empty
+            .AddHeading("app30", 1)
+            .AddRaw(
+@"This is part of [the subsystem1](../../docs/explain-subsystem1.md).
+
+We're using [My Library](../../../../shared/dotnet/MyLibrary/docs/README.md) in this app.")
+            .AddParagraphs(2));
+        folder = AddFolder("backend/subsystem1/app30/src");
+        AddFile(folder, "app30.cs", "");
+
+        folder = AddFolder("shared");
+        folder = AddFolder("shared/dotnet");
+        folder = AddFolder("shared/dotnet/MyLibrary");
+        folder = AddFolder("shared/dotnet/MyLibrary/docs");
+        AddFile(folder, "README.md", string.Empty
+            .AddHeading("My Library", 1)
+            .AddParagraphs(2));
+        folder = AddFolder("shared/dotnet/MyLibrary/src");
+        AddFile(folder, "MyLogic.cs", "");
+
+        folder = AddFolder("tools");
+        folder = AddFolder("tools/system-copilot");
+        folder = AddFolder("tools/system-copilot/docs");
+        AddFile(folder, "README.md", string.Empty
+            .AddHeading("The system copilot", 1)
+            .AddParagraphs(2)
+            .AddRaw("Go to [usage][#usage]")
+            .AddHeading("Usage", 2)
+            .AddRaw("You can use the tool like this:")
+            .AddRaw(
+@"```shell
+system-copilot -q ""provide your question here""
+```"));
+        folder = AddFolder("tools/system-copilot/SRC");
+        AddFile(folder, "system-copilot.cs", "");
+    }
+
+    public string AddFolder(string relativePath)
+    {
+        var fullPath = Path.Combine(Root, relativePath).NormalizePath();
+        Files.Add(fullPath, string.Empty);
+        return relativePath;
+    }
+
+    public void AddFile(string folderPath, string filename, string content)
+    {
+        var fullPath = Path.Combine(Root, folderPath, filename).NormalizePath();
+        Files.Add(fullPath, content.NormalizeContent());
+    }
+
+    public void Delete(string path)
+    {
+        Files.Remove(GetFullPath(path));
+    }
+
+    public void Delete(string[] paths)
+    {
+        foreach (var path in paths)
+        {
+            Files.Remove(GetFullPath(path));
+        }
+    }
+
+    public bool ExistsFileOrDirectory(string path)
+    {
+        string fullPath = GetFullPath(path);
+        return Root.Equals(fullPath, StringComparison.OrdinalIgnoreCase) || Files.ContainsKey(fullPath);
+    }
+
+    public IEnumerable GetFiles(string root, List includes, List? excludes)
+    {
+        string fullRoot = GetFullPath(root);
+
+        List rgInc = includes.Select(x => GlobToRegex(root, x)).ToList();
+        List rgExc = [];
+        if (excludes is not null)
+        {
+            rgExc = excludes.Select(x => GlobToRegex(root, x)).ToList();
+        }
+
+        List files = [];
+
+        var filesNoFolders = Files.Where(x => !string.IsNullOrEmpty(x.Value));
+        foreach (var file in filesNoFolders)
+        {
+            string selection = string.Empty;
+            // see if it matches any of the include patterns
+            foreach (string pattern in rgInc)
+            {
+                if (Regex.Match(file.Key, pattern).Success)
+                {
+                    // yes, so we're done here
+                    selection = file.Key;
+                    break;
+                }
+            }
+
+            if (!string.IsNullOrEmpty(selection))
+            {
+                // see if it's excluded by any pattern
+                foreach (string pattern in rgExc)
+                {
+                    if (Regex.Match(file.Key, pattern).Success)
+                    {
+                        // yes, so we can skip this one
+                        selection = string.Empty;
+                        break;
+                    }
+                }
+            }
+
+            if (!string.IsNullOrEmpty(selection))
+            {
+                // still have a selection, so add it to the list.
+                files.Add(selection);
+            }
+        }
+
+        return files;
+    }
+
+    public string GetFullPath(string path)
+    {
+        if (Path.IsPathRooted(path))
+        {
+            return path.NormalizePath();
+        }
+        else
+        {
+            return Path.Combine(Root, path).NormalizePath();
+        }
+    }
+
+    public string GetRelativePath(string relativeTo, string path)
+    {
+        return Path.GetRelativePath(relativeTo, path).NormalizePath();
+    }
+
+    public IEnumerable GetDirectories(string folder)
+    {
+        return Files.Where(x => x.Value == string.Empty &&
+                                x.Key.StartsWith(GetFullPath(folder)) &&
+                                !x.Key.Substring(Math.Min(GetFullPath(folder).Length + 1, x.Key.Length)).Contains("/") &&
+                                !x.Key.Equals(GetFullPath(folder), StringComparison.OrdinalIgnoreCase))
+            .Select(x => x.Key).ToList();
+    }
+
+    public string ReadAllText(string path)
+    {
+        string ipath = GetFullPath(path);
+        if (Files.TryGetValue(ipath, out var content) && !string.IsNullOrEmpty(content))
+        {
+            return content.NormalizeContent();
+        }
+
+        throw new FileNotFoundException($"File not found: '{path}'");
+    }
+
+    public string[] ReadAllLines(string path)
+    {
+        if (Files.TryGetValue(GetFullPath(path), out var content) && !string.IsNullOrEmpty(content))
+        {
+            return content.NormalizeContent().Split("\n");
+        }
+
+        throw new FileNotFoundException($"File not found: '{path}'");
+    }
+
+    public void WriteAllText(string path, string content)
+    {
+        string ipath = GetFullPath(path);
+        if (Files.TryGetValue(ipath, out var x))
+        {
+            Files.Remove(ipath);
+        }
+        Files.Add(ipath, content!.NormalizeContent());
+    }
+
+    public Stream OpenRead(string path)
+    {
+        return new MemoryStream(Encoding.UTF8.GetBytes(ReadAllText(path)));
+    }
+
+    public void Copy(string source, string destination)
+    {
+        string file = Path.GetFileName(destination);
+        string path = Path.GetDirectoryName(destination)!.NormalizePath();
+
+        if (!ExistsFileOrDirectory(source))
+        {
+            throw new FileNotFoundException($"Source file {source} not found");
+        }
+
+        EnsurePath(path);
+
+        if (!ExistsFileOrDirectory(destination))
+        {
+            string content = ReadAllText(source);
+            AddFile(path, file, content);
+        }
+    }
+
+    public void DeleteFolder(string path)
+    {
+        Delete(path);
+    }
+
+    public string GlobToRegex(string root, string input)
+    {
+        // replace **/*. where  is the extension. E.g. **/*.md
+        string pattern = @"\*\*\\\/\*\\\.(?\w+)";
+        if (Regex.Match(input, pattern).Success)
+        {
+            return root.TrimEnd('/') + "/" + Regex.Replace(input, pattern, @".+\/*\.${ext}$");
+        }
+
+        // replace **/** where  can be any test. E.g. **/*.Test.*
+        pattern = @"\*\*\/\*(?.+)\*";
+        if (Regex.Match(input, pattern).Success)
+        {
+            return Regex.Replace(input, pattern, ".+${part}.+$");
+        }
+
+        // replace **
+        pattern = @"\*\*";
+        if (Regex.Match(input, pattern).Success)
+        {
+            return root.TrimEnd('/') + Regex.Replace(input, pattern, ".*/*");
+        }
+
+        // replace *. where  is the extension. E.g. *.md
+        pattern = @"\*\.(?\w+)$";
+        if (Regex.Match(input, pattern).Success)
+        {
+            return root.TrimEnd('/') + "/" + Regex.Replace(input, pattern, @"[^\/\\]*\.${ext}$");
+        }
+
+        // replace *
+        pattern = @"\*";
+        if (Regex.Match(input, pattern).Success)
+        {
+            return root.TrimEnd('/') + "/" + Regex.Replace(input, pattern, @"[^\/\\]*$");
+        }
+
+        return input;
+    }
+
+    private void EnsurePath(string path)
+    {
+        // ensure path exists
+        string[] elms = path.Split('/');
+        string elmPath = string.Empty;
+        foreach (string elm in elms)
+        {
+            if (!string.IsNullOrEmpty(elm))
+            {
+                elmPath += "/";
+            }
+            elmPath += elm;
+            if (!ExistsFileOrDirectory(elmPath))
+            {
+                AddFolder(elmPath);
+            }
+        }
+    }
+}
+
diff --git a/src/DocAssembler/DocAssembler.Test/Helpers/MockLogger.cs b/src/DocAssembler/DocAssembler.Test/Helpers/MockLogger.cs
new file mode 100644
index 0000000..c8de8b8
--- /dev/null
+++ b/src/DocAssembler/DocAssembler.Test/Helpers/MockLogger.cs
@@ -0,0 +1,100 @@
+// 
+// Copyright (c) DocFx Companion Tools. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+// 
+using Microsoft.Extensions.Logging;
+using Moq;
+
+namespace DocAssembler.Test.Helpers;
+
+internal class MockLogger
+{
+    private readonly Mock _logger = new();
+
+    public Mock Mock => _logger;
+    public ILogger Logger => _logger.Object;
+
+    public Mock VerifyWarningWasCalled()
+    {
+        Mock.Verify(
+            x => x.Log(
+                It.Is(l => l == LogLevel.Warning),
+                It.IsAny(),
+                It.Is((v, t) => true),
+                It.IsAny(),
+                It.Is>((v, t) => true)));
+
+        return Mock;
+    }
+
+    public Mock VerifyWarningWasCalled(string expectedMessage)
+    {
+        Func state = (v, t) => v.ToString()!.CompareTo(expectedMessage) == 0;
+
+        Mock.Verify(
+            x => x.Log(
+                It.Is(l => l == LogLevel.Warning),
+                It.IsAny(),
+                It.Is((v, t) => state(v, t)),
+                It.IsAny(),
+                It.Is>((v, t) => true)));
+
+        return Mock;
+    }
+
+    public Mock VerifyErrorWasCalled()
+    {
+        Mock.Verify(
+            x => x.Log(
+                It.Is(l => l == LogLevel.Error),
+                It.IsAny(),
+                It.Is((v, t) => true),
+                It.IsAny(),
+                It.Is>((v, t) => true)));
+
+        return Mock;
+    }
+
+    public Mock VerifyErrorWasCalled(string expectedMessage)
+    {
+        Func state = (v, t) => v.ToString()!.CompareTo(expectedMessage) == 0;
+
+        Mock.Verify(
+            x => x.Log(
+                It.Is(l => l == LogLevel.Error),
+                It.IsAny(),
+                It.Is((v, t) => state(v, t)),
+                It.IsAny(),
+                It.Is>((v, t) => true)));
+
+        return Mock;
+    }
+
+    public Mock VerifyCriticalWasCalled()
+    {
+        Mock.Verify(
+            x => x.Log(
+                It.Is(l => l == LogLevel.Critical),
+                It.IsAny(),
+                It.Is((v, t) => true),
+                It.IsAny(),
+                It.Is>((v, t) => true)));
+
+        return Mock;
+    }
+
+    public Mock VerifyCriticalWasCalled(string expectedMessage)
+    {
+        Func state = (v, t) => v.ToString()!.CompareTo(expectedMessage) == 0;
+
+        Mock.Verify(
+            x => x.Log(
+                It.Is(l => l == LogLevel.Critical),
+                It.IsAny(),
+                It.Is((v, t) => state(v, t)),
+                It.IsAny(),
+                It.Is>((v, t) => true)));
+
+        return Mock;
+    }
+}
diff --git a/src/DocAssembler/DocAssembler.Test/InventoryActionTests.cs b/src/DocAssembler/DocAssembler.Test/InventoryActionTests.cs
new file mode 100644
index 0000000..dc017c2
--- /dev/null
+++ b/src/DocAssembler/DocAssembler.Test/InventoryActionTests.cs
@@ -0,0 +1,307 @@
+// 
+// Copyright (c) DocFx Companion Tools. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+// 
+using Bogus;
+using DocAssembler.Actions;
+using DocAssembler.Configuration;
+using DocAssembler.Test.Helpers;
+using FluentAssertions;
+using Microsoft.Extensions.Logging;
+
+namespace DocAssembler.Test;
+
+public class InventoryActionTests
+{
+    private Faker _faker = new();
+    private MockFileService _fileService = new();
+    private MockLogger _mockLogger = new();
+    private ILogger _logger;
+
+    private string _workingFolder = string.Empty;
+    private string _outputFolder = string.Empty;
+
+    public InventoryActionTests()
+    {
+        _fileService.FillDemoSet();
+        _logger = _mockLogger.Logger;
+
+        _workingFolder = _fileService.Root;
+        _outputFolder = Path.Combine(_fileService.Root, "out");
+    }
+
+    [Fact]
+    public async void Run_StandardConfigProducesExpectedFiles()
+    {
+        // arrange
+        AssembleConfiguration config = GetStandardConfiguration();
+        InventoryAction action = new(_workingFolder, config, _fileService, _logger);
+        // all files in .docfx and docs-children
+        var expected = _fileService.Files.Where(x => !string.IsNullOrEmpty(x.Value) &&
+                                                     (x.Key.Contains("/.docfx/") || x.Key.Contains("/docs/")));
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        action.Files.Should().HaveCount(expected.Count());
+    }
+
+    [Fact]
+    public async void Run_MinimimalRawConfigProducesExpectedFiles()
+    {
+        // arrange
+        AssembleConfiguration config = new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            Content =
+            [
+                new Content
+                    {
+                        SourceFolder = ".docfx",
+                        Files = { "**" },
+                        RawCopy = true,         // just copy the content
+                    }
+                ],
+        };
+        InventoryAction action = new(_workingFolder, config, _fileService, _logger);
+        // all files in .docfx
+        int expected = _fileService.Files.Count(x => !string.IsNullOrEmpty(x.Value) && x.Key.Contains("/.docfx/"));
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        action.Files.Should().HaveCount(expected);
+    }
+
+    [Fact]
+    public async void Run_MinimalRawConfigWithDoubleContent_ShouldFail()
+    {
+        // arrange
+        AssembleConfiguration config = new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            Content =
+            [
+                new Content
+                {
+                    SourceFolder = ".docfx",
+                    Files = { "**" },
+                    RawCopy = true,         // just copy the content
+                },
+                new Content
+                {
+                    SourceFolder = ".docfx", // same content and destination should fail.
+                    Files = { "**" },
+                    RawCopy = true,
+                }
+            ],
+        };
+        InventoryAction action = new(_workingFolder, config, _fileService, _logger);
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Error);
+    }
+
+    [Fact]
+    public async void Run_MinimalRawConfig_WithGlobalChangedPaths()
+    {
+        // arrange
+        AssembleConfiguration config = new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            UrlReplacements =
+            [
+                new Replacement
+                {
+                    Expression = @"/[Ii]mages/",
+                    Value = "/assets/"
+                }
+            ],
+            Content =
+            [
+                new Content
+                    {
+                        SourceFolder = ".docfx",
+                        Files = { "**" },
+                        RawCopy = true,         // just copy the content
+                    }
+                ],
+        };
+        InventoryAction action = new(_workingFolder, config, _fileService, _logger);
+        // all files in .docfx
+        int expected = _fileService.Files.Count(x => !string.IsNullOrEmpty(x.Value) && x.Key.Contains("/.docfx/"));
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        action.Files.Should().HaveCount(expected);
+        var assets = action.Files.Where(x => x.SourcePath.Contains("/images"));
+        assets.Should().HaveCount(1);
+
+        string expectedPath = assets.First().SourcePath
+            .Replace($"{_fileService.Root}/.docfx", $"{_fileService.Root}/{config.DestinationFolder}")
+            .Replace("/images/", "/assets/");
+        assets.First().DestinationPath.Should().Be(expectedPath);
+    }
+
+    [Fact]
+    public async void Run_MinimalRawConfig_WithContentChangedPaths()
+    {
+        // arrange
+        AssembleConfiguration config = new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            Content =
+            [
+                new Content
+                    {
+                        SourceFolder = ".docfx",
+                        Files = { "**" },
+                        RawCopy = true,         // just copy the content
+                        UrlReplacements =
+                        [
+                            new Replacement
+                            {
+                                Expression = @"/[Ii]mages/",
+                                Value = "/assets/"
+                            }
+                        ],
+                    }
+                ],
+        };
+        InventoryAction action = new(_workingFolder, config, _fileService, _logger);
+        // all files in .docfx
+        int expected = _fileService.Files.Count(x => !string.IsNullOrEmpty(x.Value) && x.Key.Contains("/.docfx/"));
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        action.Files.Should().HaveCount(expected);
+        var assets = action.Files.Where(x => x.SourcePath.Contains("/images"));
+        assets.Should().HaveCount(1);
+
+        string expectedPath = assets.First().SourcePath
+            .Replace($"{_fileService.Root}/.docfx", $"{_fileService.Root}/{config.DestinationFolder}")
+            .Replace("/images/", "/assets/");
+        assets.First().DestinationPath.Should().Be(expectedPath);
+    }
+
+    [Fact]
+    public async void Run_MinimalRawConfig_WithContentOverruledNotChangedPaths()
+    {
+        // arrange
+        AssembleConfiguration config = new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            UrlReplacements =
+            [
+                new Replacement
+                {
+                    Expression = @"/[Ii]mages/",
+                    Value = "/assets/"
+                }
+            ],
+            Content =
+            [
+                new Content
+                    {
+                        SourceFolder = ".docfx",
+                        Files = { "**" },
+                        RawCopy = true,         // just copy the content
+                        UrlReplacements = [],   // this overrides the global replacement
+                    }
+                ],
+        };
+        InventoryAction action = new(_workingFolder, config, _fileService, _logger);
+        // all files in .docfx
+        int expected = _fileService.Files.Count(x => !string.IsNullOrEmpty(x.Value) && x.Key.Contains("/.docfx/"));
+
+        // act
+        var ret = await action.RunAsync();
+
+        // assert
+        ret.Should().Be(ReturnCode.Normal);
+        action.Files.Should().HaveCount(expected);
+        var assets = action.Files.Where(x => x.SourcePath.Contains("/images"));
+        assets.Should().HaveCount(1);
+
+        string expectedPath = assets.First().SourcePath
+            .Replace($"{_fileService.Root}/.docfx", $"{_fileService.Root}/{config.DestinationFolder}");
+        assets.First().DestinationPath.Should().Be(expectedPath);
+    }
+
+    private AssembleConfiguration GetStandardConfiguration()
+    {
+        return new AssembleConfiguration
+        {
+            DestinationFolder = "out",
+            ExternalFilePrefix = "https://github.com/example/blob/main/",
+            UrlReplacements =
+            [
+                new Replacement
+                {
+                    Expression = @"/[Dd]ocs/",
+                    Value = "/"
+                }
+            ],
+            ContentReplacements =
+            [
+                new Replacement
+                {
+                    Expression = @"(?
[$\s])AB#(?[0-9]{3,6})",
+                    Value = @"${pre}[AB#${id}](https://dev.azure.com/MyCompany/MyProject/_workitems/edit/${id})"
+                },
+                new Replacement     // Remove markdown style table of content
+                {
+                    Expression = @"\[\[_TOC_\]\]",
+                    Value = ""
+                }
+            ],
+            Content =
+            [
+                new Content
+                    {
+                        SourceFolder = ".docfx",
+                        Files = { "**" },
+                        RawCopy = true,         // just copy the content
+                        UrlReplacements = []    // reset URL replacements
+                    },
+                    new Content
+                    {
+                        SourceFolder = "docs",
+                        DestinationFolder = "general",
+                        Files = { "**" },
+                    },
+                    new Content
+                    {
+                        SourceFolder = "shared",    // part of general docs
+                        DestinationFolder = "general/shared",
+                        Files = { "**/docs/**" },
+                    },
+                    new Content
+                    {
+                        SourceFolder = "tools",     // part of general docs
+                        DestinationFolder = "general/tools",
+                        Files = { "**/docs/**" },
+                    },
+                    new Content
+                    {
+                        SourceFolder = "backend",
+                        DestinationFolder = "services", // change name to services
+                        Files = { "**/docs/**" },
+                    },
+                ],
+        };
+    }
+}
diff --git a/src/DocAssembler/DocAssembler.sln b/src/DocAssembler/DocAssembler.sln
index 6c56fb3..496c49d 100644
--- a/src/DocAssembler/DocAssembler.sln
+++ b/src/DocAssembler/DocAssembler.sln
@@ -3,7 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.11.35431.28
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DocAssembler", "DocAssembler\DocAssembler.csproj", "{20348289-FB98-4EE3-987D-576E3C568EB3}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DocAssembler", "DocAssembler\DocAssembler.csproj", "{20348289-FB98-4EE3-987D-576E3C568EB3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DocAssembler.Test", "DocAssembler.Test\DocAssembler.Test.csproj", "{BA44E0E9-6D85-4185-99BA-8697A02663A1}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -15,6 +17,10 @@ Global
 		{20348289-FB98-4EE3-987D-576E3C568EB3}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{20348289-FB98-4EE3-987D-576E3C568EB3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{20348289-FB98-4EE3-987D-576E3C568EB3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BA44E0E9-6D85-4185-99BA-8697A02663A1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BA44E0E9-6D85-4185-99BA-8697A02663A1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BA44E0E9-6D85-4185-99BA-8697A02663A1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BA44E0E9-6D85-4185-99BA-8697A02663A1}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
diff --git a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs
index 3ad02da..7cff8b4 100644
--- a/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs
+++ b/src/DocAssembler/DocAssembler/Actions/AssembleAction.cs
@@ -2,6 +2,7 @@
 // Copyright (c) DocFx Companion Tools. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 // 
+using System;
 using System.Text;
 using System.Text.RegularExpressions;
 using DocAssembler.Configuration;
@@ -56,7 +57,7 @@ public Task RunAsync()
                 var updates = file.Links
                     .Where(x => !x.OriginalUrl.Equals(x.DestinationRelativeUrl ?? x.DestinationFullUrl, StringComparison.Ordinal))
                     .OrderBy(x => x.UrlSpanStart);
-                if (file.IsMarkdown && (updates.Any() || file.ContentSet?.ContentReplacements is not null))
+                if (file.IsMarkdown && (updates.Any() || _config.ContentReplacements is not null || file.ContentSet?.ContentReplacements is not null))
                 {
                     var markdown = _fileService.ReadAllText(file.SourcePath);
                     StringBuilder sb = new StringBuilder();
diff --git a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs
index 98062fc..12aac2f 100644
--- a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs
+++ b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs
@@ -2,7 +2,6 @@
 // Copyright (c) DocFx Companion Tools. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 // 
-using System.Linq.Expressions;
 using DocAssembler.Configuration;
 using DocAssembler.FileService;
 using DocAssembler.Utils;
@@ -43,19 +42,19 @@ public ConfigInitAction(
     /// Run the action.
     ///
/// 0 on success, 1 on warning, 2 on error. - public async Task RunAsync() + public Task RunAsync() { ReturnCode ret = ReturnCode.Normal; try { string path = Path.Combine(_outFolder, CONFIGFILENAME); - if (File.Exists(path)) + if (_fileService!.ExistsFileOrDirectory(path)) { _logger.LogError($"*** ERROR: '{path}' already exists. We don't overwrite."); // indicate we're done with an error - return ReturnCode.Error; + return Task.FromResult(ReturnCode.Error); } var config = new AssembleConfiguration @@ -91,7 +90,7 @@ public async Task RunAsync() ], }; - await File.WriteAllTextAsync(path, SerializationUtil.Serialize(config)); + _fileService.WriteAllText(path, SerializationUtil.Serialize(config)); _logger.LogInformation($"Initial configuration saved in '{path}'"); } catch (Exception ex) @@ -100,6 +99,6 @@ public async Task RunAsync() ret = ReturnCode.Error; } - return ret; + return Task.FromResult(ret); } } diff --git a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs index e12a0fa..d64b5ac 100644 --- a/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/InventoryAction.cs @@ -61,11 +61,14 @@ public Task RunAsync() try { ret = GetAllFiles(); - ret &= ValidateFiles(); + if (ret != ReturnCode.Error) + { + ret = ValidateFiles(); + } if (ret != ReturnCode.Error) { - ret &= UpdateLinks(); + ret = UpdateLinks(); // log result of inventory (verbose) foreach (var file in Files) diff --git a/src/DocAssembler/DocAssembler/Configuration/Content.cs b/src/DocAssembler/DocAssembler/Configuration/Content.cs index 9a52de3..ec0d432 100644 --- a/src/DocAssembler/DocAssembler/Configuration/Content.cs +++ b/src/DocAssembler/DocAssembler/Configuration/Content.cs @@ -2,6 +2,7 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using System.Diagnostics.CodeAnalysis; using System.Text.Json.Serialization; namespace DocAssembler.Configuration; diff --git a/src/DocAssembler/DocAssembler/DocAssembler.csproj b/src/DocAssembler/DocAssembler/DocAssembler.csproj index 5ada01e..bcec377 100644 --- a/src/DocAssembler/DocAssembler/DocAssembler.csproj +++ b/src/DocAssembler/DocAssembler/DocAssembler.csproj @@ -24,7 +24,7 @@ - + diff --git a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs index b4832c0..6bd8910 100644 --- a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs +++ b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs @@ -78,6 +78,7 @@ public List GetLocalHyperlinks(string root, string filePath) // e.g. a link "..\..\somefile.md" resolves in "....\somefile.md" // we fix that here. This will probably not be fixed in the markdig // library, as you shouldn't use backslash, but Unix-style slash. + link.OriginalUrl = markdown.Substring(link.UrlSpanStart, link.UrlSpanLength); link.Url = markdown.Substring(link.UrlSpanStart, link.UrlSpanLength); } @@ -98,9 +99,10 @@ public List GetLocalHyperlinks(string root, string filePath) // we want to know that the link is not starting with a # for local reference. // if local reference, return the filename otherwise the calculated path. - string destFullPath = pos != 0 ? - Path.Combine(Path.GetDirectoryName(link.FilePath)!, link.UrlWithoutTopic) : link.FilePath; - link.UrlFullPath = _fileService.GetFullPath(destFullPath); + if (pos != 0) + { + link.UrlFullPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(link.FilePath)!.NormalizePath(), link.UrlWithoutTopic)).NormalizePath(); + } } else { diff --git a/src/DocAssembler/DocAssembler/FileService/MarkdownLink.cs b/src/DocAssembler/DocAssembler/FileService/MarkdownLink.cs deleted file mode 100644 index 8b4267a..0000000 --- a/src/DocAssembler/DocAssembler/FileService/MarkdownLink.cs +++ /dev/null @@ -1,16 +0,0 @@ -// -// Copyright (c) DocFx Companion Tools. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. -// -namespace DocAssembler.FileService; - -/// -/// Markdown link in document. -/// -public sealed record MarkdownLink -{ - /// - /// Gets or sets the URL. - /// - public string Url { get; set; } = string.Empty; -} diff --git a/src/DocAssembler/DocAssembler/README.md b/src/DocAssembler/DocAssembler/README.md deleted file mode 100644 index dcdfeaf..0000000 --- a/src/DocAssembler/DocAssembler/README.md +++ /dev/null @@ -1,355 +0,0 @@ -# Table of Contents (TOC) generator for DocFX - -This tool allow to generate a yaml compatible `toc.yml` file for DocFX. - -## Usage - -```text -DocAssembler [options] - -Options: - -d, --docfolder (REQUIRED) The root folder of the documentation. - -o, --outfolder The output folder for the generated table of contents - file. Default is the documentation folder. - -v, --verbose Show verbose messages of the process. - -s, --sequence Use .order files per folder to define the sequence of - files and directories. Format of the file is filename - without extension per line. - -r, --override Use .override files per folder to define title overrides - for files and folders. Format of the file is filename - without extension or directory name followed by a - semi-column followed by the custom title per line. - -g, --ignore Use .ignore files per folder to ignore directories. - Format of the file is directory name per line. - --indexing When to generated an index.md for a folder. - NoDefault - When no index.md or readme.md found. - NoDefaultMulti - When no index.md or readme.md found and - multiple files. - EmptyFolders - For empty folders. - NotExists - When no index found. - NotExistMulti - When no index and multiple files. - [default: Never] - --folderRef Strategy for folder-entry references. - None - Never reference anything. - Index - Index.md only if exists. - IndexReadme - Index.md or readme.md if exists. - First - First file in folder if any exists. - [default: First] - --ordering How to order items in a folder. - All - Folders and files combined. - FoldersFirst - Folders first, then files. - FilesFirst - Files first, then folders. [default: All] - -m, --multitoc Indicates how deep in the tree toc files should be - generated for those folders. A depth of 0 is the root - only (default behavior). - --camelCase Use camel casing for titles. - --version Show version information - -?, -h, --help Show help and usage information -``` - -Return values: - 0 - succesfull. - 1 - some warnings, but process could be completed. - 2 - a fatal error occurred. - -## Warnings, errors and verbose - -If the tool encounters situations that might need some action, a warning is written to the output. The table of contents is still created. If the tool encounters an error, an error message is written to the output. The table of contents will not be created. - -If you want to trace what the tool is doing, use the `-v or --verbose` flag to output all details of processing the files and folders and creating the table of contents. - -## Overall process - -The overall process of this tool is: - -1. Content inventory - retrieve all folders and files (`*.md` and `*swagger.json`) in the given documentation folder. Flags `-s | --sequence`, `-r | --override` and `-g | --ignore` are processed here to read setting files in the hierarchy. -2. Ensure indexing - validate structure with given settings. Depending on the `--indexing` flag automated `index.md` files are added where necessary. -3. Generate the table of contents - generate the `toc.yml` file(s). For folders it can be indicated if they should have a reference into child files using the `--folderRef` flag. Using the `--ordering` flag the ordering of directories and files can be defined. In this step the `-m | --multitoc ` flag is evaluated and processed on generation. - -### Title of directories and files - -For directories the name of the directory is used by default, where the first character is uppercased and special characters (`[`, `]`, `:`, \`,`\`, `{`, `}`, `(`, `)`, `*`, `/`) are removed and `-`, `_` and multiple spaces are replaced by a single space. - -For markdown files the first level-1 heading is taken as title. For swagger files the title and version are taken as title. On error the file name without extension is taken and processed the same way as the name of a directory. - -The `.override` setting file can be used to override this behavior. See [Defining title overrides with `.override`](#defining-title-overrides-with-override). - -## Folder settings - -Folder settings can be provided on ordering directories and files, ignore directories and override titles of files. Flags `-s | --sequence`, `-r | --override` and `-g | --ignore` are processed here to read setting files in the hierarchy. - -### Defining the order with `.order` - -If the `-s | --sequence` parameter is provided, the tool will inspect folders if a `.order` file exists and use that to determine the order of files and directories. The `.order` file is just a list of file- and/or directory-names, *case-sensitive* without file extensions. Also see the [Azure DevOps WIKI documentation on this file](https://docs.microsoft.com/en-us/azure/devops/project/wiki/wiki-file-structure?view=azure-devops#order-file). - -A sample `.order` file looks like this: - -```text -getting-started -working-agreements -developer -``` - -Ordering of directories and files in a folder is influenced by the `-s | --sequence` flag in combination with the `.order` file in that directory, combined with the (optional) `--ordering` flag. Also see [Ordering](#ordering). - -### Defining directories to ignore with `.ignore` - -If the `-g | --ignore` parameter is provided, the tool will inspect folders if a `.ignore` file exists and use that to ignore directories. The `.ignore` file is just a list of file- and/or directory-names, *case-sensitive* without file extensions. - -A sample `.ignore` file looks like this: - -```text -node_modules -bin -``` - -It only applies to the folder it's in, not for other subfolders under that folder. - -### Defining title overrides with `.override` - -If the `-r | --override` parameter is provided, the tool will inspect folders if a `.override` file exists and use that for overrides of file or directory titles as they will show in the generated `toc.yml`. The `.override` file is a list of file- and/or directory-names, *case-sensitive* without file extensions, followed by a semi-column, followed by the title to use. - -For example, if the folder name is `introduction`, the default behavior will be to create the name `Introduction`. If you want to call it `To start with`, you can use overrides, like in the following example: - -```text -introduction;To start with -working-agreements;All working agreements of all teams -``` - -The title for an MD-file is taken from the H1-header in the file. The title for a directory is the directory-name, but cleanup from special characters and the first character in capitals. - -## Automatic generating `index.md` files - -If the `-indexing ` parameter is provided the `method` defines the conditions for generating an `index.md` file. The options are: - -* `Never` - never generate an `index.md`. This is the default. -* `NoDefault` - generate an `index.md` when no `index.md` or `readme.md` is found in a folder. -* `NoDefaultMulti` - generate an `index.md` when no `index.md` or `readme.md` is found in a folder and there are 2 or more files. -* `NotExists` - generate an `index.md` when no `index.md` file is found in a folder. -* `NotExistsMulti` - generate an `index.md` when no `index.md` file is found in a folder and there are 2 or more files. -* `EmptyFolders` - generate an `index.md` when a folder doesn't contain any files. - -### Template for generating an `index.md` - -When an `index.md` file is generated, this is done by using a [Liquid template](https://shopify.github.io/liquid/). The tool contains a *default template*: - -```liquid -# {{ current.DisplayName }} - -{% comment -%}Looping through all the files and show the display name.{%- endcomment -%} -{% for file in current.Files -%} -{%- if file.IsMarkdown -%} -* [{{ file.DisplayName }}]({{ file.Name }}) -{% endif -%} -{%- endfor %} -``` - -This results in a markdown file like this: - -```markdown -# Brasil - -* [Nova Friburgo](nova-friburgo.md) -* [Rio de Janeiro](rio-de-janeiro.md) -* [Sao Paulo](sao-paulo.md) -``` - -You can also provide a customized template to be used. The ensure indexing process will look for a file with the name `.index.liquid` in the folder where an `index.md` needs to be generated. If it doesn't exist in that folder it's traversing all parent folders up to the root and until a `.index.liquid` file is found. - -In the template access is provided to this information: - -* `current` - this is the current folder that needs an `index.md` file of type `FolderData`. -* `root` - this is the root folder of the complete hierarchy of the documentation of type `FolderData`. - -#### `FolderData` class - -| Property | Description | -| -------------- | ------------------------------------------------------------ | -| `Name` | Folder name from disk | -| `DisplayName` | Title of the folder | -| `Path` | Full path of the folder | -| `Sequence` | Sequence number from the `.order` file or `int.MaxValue` when not defined. | -| `RelativePath` | Relative path of the folder from the root of the documentation. | -| `Parent` | Parent folder. When `null` it's the root folder. | -| `Folders` | A list of `FolderData` objects for the sub-folders in this folder. | -| `Files` | A list of `FileData` objects for the files in this folder. | -| `HasIndex` | A `boolean` indicating whether this folder contains an `index.md` | -| `Index` | The `FileData` object of the `index.md` in this folder if it exists. If it doesn't exists this will be `null`. | -| `HasReadme` | A `boolean` indicating whether this folder contains an `README.md` | -| `Readme` | The `FileData` object of the `README.md` in this folder if it exists. If it doesn't exists this will be `null`. | - -#### `FileData` class - -| Property | Description | -| -------------- | ------------------------------------------------------------ | -| `Name` | Filename including the extension | -| `DisplayName` | Title of the file. | -| `Path` | Full path of the file | -| `Sequence` | Sequence number from the `.order` file or `int.MaxValue` when not defined. | -| `RelativePath` | Relative path of the file from the root of the documentation. | -| `Parent` | Parent folder. | -| `IsMarkdown` | A `boolean` indicating whether this file is a markdown file. | -| `IsSwagger` | A `boolean` indicating whether this file is a Swagger JSON file. | -| `IsIndex` | A `boolean` indicating whether this file is an `index.md` file. | -| `IsReadme` | A `boolean` indicating whether this file is a `README.md` file. | - -For more information on how to use Liquid logic, see the article [Using Liquid for text-based templates with .NET | by Martin Tirion | Medium](https://mtirion.medium.com/using-liquid-for-text-base-templates-with-net-80ae503fa635) and the [Liquid reference](https://shopify.github.io/liquid/basics/introduction/). - -Liquid, by design, is very forgiving. If you reference an object or property that doesn't exist, it will render to an empty string. But if you introduce language errors (missing `{{` for instance) an error is thrown, the error is in the output of the tool but will not crash the tool, but will be resulting in error code 1 (warning). In the case of an error like this, no `index.md` is generated. - -## Ordering - -There are these options for ordering directories and folders: - -* `All` - order all directories and files by sequence, then by title. -* `FoldersFirst` - order all directories first, then the files. Ordering is for each of them done by sequence, then by title. -* `FilesFirst` - order all files first, then the folders. Ordering is for each of them done by sequence, then by title. - -For all of these options the `.order` file can be used when it exists and the `-s | --sequence` flag is used. The line in the `.order` file determines the sequence of a file or directory. So, the first entry results in sequence 1. In all other cases a folder or file has an equal sequence of `int.MaxValue`. - -By default the ordering of files is applied where the `index.md` is first and the `README.md` is second, optionally followed by the settings from the `.order` file. This behavior can only be overruled by adding `index` and/or `readme` to a `.order` file and use of the `-s | --sequence` flag. - -> [!NOTE] -> -> `README` and `index` are always validated **case-sensitive** to make sure they are ordered correctly. All other file names and directory names are matched **case-insensitive**. - -## Folder referencing - -The table of content is constructed from the folders and files. For folders there are various strategies to determine if it will have a reference: - -* `None` - no reference for all folders. -* `Index` - reference the `index.md` in the folder if it exists. -* `IndexReadme` - reference the `index.md` if it exists, otherwise reference the `README.md` if it exists. -* `First` - reference the first file in the folder after [ordering](#ordering) has been applied. - -When using DocFx to generate the website, folders with no reference will just be entries in the hive that can be opened and closed. The UI will determine what will be showed as content. - -## Multiple table of content files - -The default for this tool is to generate only one `toc.yml` file in the root of the output directory. But with a large hierarchy, this file can get pretty large. In that case it might be easier to have a few `toc.yml` files per level to have multiple, smaller `toc.yml` files. - -The `-m | --multitoc` option will control how far down the hierarchy `toc.yml` files are generated. Let's explain this feature by an example hierarchy: - -```text -📂docs - 📄README.md - 📂continents - 📄index.md - 📂americas - 📄README.md - 📄extra-facts.md - 📂brasil - 📄README.md - 📄nova-friburgo.md - 📄rio-de-janeiro.md - 📂united-states - 📄los-angeles.md - 📄new-york.md - 📄washington.md - 📂europe - 📄README.md - 📂germany - 📄berlin.md - 📄munich.md - 📂netherlands - 📄amsterdam.md - 📄rotterdam.md - 📂vehicles - 📄index.md - 📂cars - 📄README.md - 📄audi.md - 📄bmw.md -``` - -### Default behavior or depth=0 - -By default, when the `depth` is `0` (or the option is omitted), only one `toc.yml` file is generated in the root of the output folder containing the complete hierarchy of folders and files. For the example hierarchy it would look like this: - -```yaml -# This is an automatically generated file -- name: Multi toc example - href: README.md -- name: Continents - href: continents/index.md - items: - - name: Americas - href: continents/americas/README.md - items: - - name: Americas Extra Facts - href: continents/americas/extra-facts.md - - name: Brasil - href: continents/americas/brasil/README.md - items: - - name: Nova Friburgo - href: continents/americas/brasil/nova-friburgo.md - - name: Rio de Janeiro - href: continents/americas/brasil/rio-de-janeiro.md - - name: Los Angeles - href: continents/americas/united-states/los-angeles.md - items: - - name: New York - href: continents/americas/united-states/new-york.md - - name: Washington - href: continents/americas/united-states/washington.md - - name: Europe - href: continents/europe/README.md - items: - - name: Amsterdam - href: continents/europe/netherlands/amsterdam.md - items: - - name: Rotterdam - href: continents/europe/netherlands/rotterdam.md - - name: Berlin - href: continents/europe/germany/berlin.md - items: - - name: Munich - href: continents/europe/germany/munich.md -- name: Vehicles - href: vehicles/index.md - items: - - name: Cars - href: vehicles/cars/README.md - items: - - name: Audi - href: vehicles/cars/audi.md - - name: BMW - href: vehicles/cars/bmw.md - -``` - -### Behavior with depth=1 or more - -When a `depth` of `1` is given, a `toc.yml` is generated in the root of the output folder and in each sub-folder of the documentation root. The `toc.yml` in the root will only contain documents of the folder itself and references to the `toc.yml` files in the sub-folders. In our example for the root it would look like this: - -```yaml -# This is an automatically generated file -- name: Multi toc example - href: README.md -- name: Continents - href: continents/toc.yml -- name: Vehicles - href: vehicles/toc.yml -``` - -The `toc.yml` files in the sub-folders `continents` and `vehicles` will contain the complete hierarchy from that point on. For instance, for `vehicles` it will look like this: - -```yaml -# This is an automatically generated file -- name: Cars - href: cars/README.md - items: - - name: Audi - href: cars/audi.md - - name: BMW - href: cars/bmw.md -``` - -## Camel case titles - -By default titles are changed to pascal casing, meaning that the first character is capitalized. With the option `--camelCase` all titles will be changed to camel casing, meaning that the first character is lower cased. Only exception are overrides from `.override` files. - -> [!NOTE] -> -> As this rule is applied to everything, it is also applied to titles coming from Swagger-files. If this is an issue, this can be corrected for that file using an `.override` file in that folder. diff --git a/src/DocAssembler/DocAssembler/Utils/SerializationUtil.cs b/src/DocAssembler/DocAssembler/Utils/SerializationUtil.cs index 889249f..0c5ed01 100644 --- a/src/DocAssembler/DocAssembler/Utils/SerializationUtil.cs +++ b/src/DocAssembler/DocAssembler/Utils/SerializationUtil.cs @@ -6,7 +6,7 @@ namespace DocAssembler.Utils; /// /// Serialization utilities. /// -internal static class SerializationUtil +public static class SerializationUtil { /// /// Gets the JSON serializer options. From 917922a91157634e9a6948a8cad4fb13e8d1ecab Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Thu, 12 Dec 2024 20:08:46 +0100 Subject: [PATCH 10/12] added reference in main readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index bc10966..96cce15 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ This repository contains a series of tools, templates, tips and tricks to make y ## Tools +* [DocAssembler 🆕](./src/DocAssembler): assemble documentation and assets from various locations on disk and assemble them in one place. It is possible to restructure, where the links are changed to the right location. * [DocFxTocGenerator](./src/DocFxTocGenerator): generate a Table of Contents (TOC) in YAML format for DocFX. It has features like the ability to configure the order of files and the names of documents and folders. * [DocLinkChecker](./src/DocLinkChecker): validate links in documents and check for orphaned attachments in the `.attachments` folder. The tool indicates whether there are errors or warnings, so it can be used in a CI pipeline. It can also clean up orphaned attachments automatically. And it can validate table syntax. * [DocLanguageTranslator](./src/DocLanguageTranslator): allows to generate and translate automatically missing files or identify missing files in multi language pattern directories. @@ -66,6 +67,7 @@ choco install docfx-companion-tools You can as well install the tools through `dotnet tool`. ```shell +dotnet tool install DocAssembler -g dotnet tool install DocFxTocGenerator -g dotnet tool install DocLanguageTranslator -g dotnet tool install DocLinkChecker -g From 7eb4818406c881d62e02a31f207cd93bda2a32a1 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Mon, 16 Dec 2024 13:58:45 +0100 Subject: [PATCH 11/12] Changed from PR comments --- .../DocAssembler/Actions/ConfigInitAction.cs | 4 ++-- .../DocAssembler/FileService/Hyperlink.cs | 10 +++++----- src/DocAssembler/DocAssembler/Utils/LogUtil.cs | 11 +---------- src/DocAssembler/README.md | 6 ++---- 4 files changed, 10 insertions(+), 21 deletions(-) diff --git a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs index 12aac2f..c75cfab 100644 --- a/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs +++ b/src/DocAssembler/DocAssembler/Actions/ConfigInitAction.cs @@ -14,7 +14,7 @@ namespace DocAssembler.Actions; /// public class ConfigInitAction { - private const string CONFIGFILENAME = ".docassembler.json"; + private const string ConfigFileName = ".docassembler.json"; private readonly string _outFolder; @@ -48,7 +48,7 @@ public Task RunAsync() try { - string path = Path.Combine(_outFolder, CONFIGFILENAME); + string path = Path.Combine(_outFolder, ConfigFileName); if (_fileService!.ExistsFileOrDirectory(path)) { _logger.LogError($"*** ERROR: '{path}' already exists. We don't overwrite."); diff --git a/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs b/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs index e919597..ac43374 100644 --- a/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs +++ b/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs @@ -11,9 +11,9 @@ namespace DocAssembler.FileService; /// public class Hyperlink { - private static readonly char[] UriFragmentOrQueryString = new char[] { '#', '?' }; - private static readonly char[] AdditionalInvalidChars = @"\/?:*".ToArray(); - private static readonly char[] InvalidPathChars = Path.GetInvalidPathChars().Concat(AdditionalInvalidChars).ToArray(); + private static readonly char[] _uriFragmentOrQueryString = new char[] { '#', '?' }; + private static readonly char[] _additionalInvalidChars = @"\/?:*".ToArray(); + private static readonly char[] _invalidPathChars = Path.GetInvalidPathChars().Concat(_additionalInvalidChars).ToArray(); /// /// Initializes a new instance of the class. @@ -230,7 +230,7 @@ private string UrlDecode(string url) } var anchor = string.Empty; - var index = url.IndexOfAny(UriFragmentOrQueryString); + var index = url.IndexOfAny(_uriFragmentOrQueryString); if (index != -1) { anchor = url.Substring(index); @@ -249,7 +249,7 @@ private string UrlDecode(string url) var origin = parts[i]; var value = Uri.UnescapeDataString(origin); - var splittedOnInvalidChars = value.Split(InvalidPathChars); + var splittedOnInvalidChars = value.Split(_invalidPathChars); var originIndex = 0; var valueIndex = 0; for (int j = 0; j < splittedOnInvalidChars.Length; j++) diff --git a/src/DocAssembler/DocAssembler/Utils/LogUtil.cs b/src/DocAssembler/DocAssembler/Utils/LogUtil.cs index a39cf74..20ac599 100644 --- a/src/DocAssembler/DocAssembler/Utils/LogUtil.cs +++ b/src/DocAssembler/DocAssembler/Utils/LogUtil.cs @@ -24,16 +24,7 @@ internal static class LogUtil /// When an unknown log level is given. public static ILoggerFactory GetLoggerFactory(LogLevel logLevel1) { - var serilogLevel = logLevel1 switch - { - LogLevel.Critical => LogEventLevel.Fatal, - LogLevel.Error => LogEventLevel.Error, - LogLevel.Warning => LogEventLevel.Warning, - LogLevel.Information => LogEventLevel.Information, - LogLevel.Debug => LogEventLevel.Debug, - LogLevel.Trace => LogEventLevel.Verbose, - _ => throw new ArgumentOutOfRangeException(nameof(logLevel1)), - }; + var serilogLevel = (LogEventLevel)logLevel1; var serilog = new LoggerConfiguration() .MinimumLevel.Is(serilogLevel) diff --git a/src/DocAssembler/README.md b/src/DocAssembler/README.md index aa6cb8f..93d1712 100644 --- a/src/DocAssembler/README.md +++ b/src/DocAssembler/README.md @@ -127,7 +127,7 @@ As we don't want to find a link like `[AB#1234](https://...)`, we look for all A > [!NOTE] > -> As the expression is configured in a string in a JSON file, special characters like back-slashes need to be escaped by an (extra) back-slash. +> As the expression is configured in a string in a JSON file, special characters like back-slashes need to be escaped by an (extra) back-slash like you see in the example above, where `\s` is escaped with an extra `\`. The second part is to get the numbers after the AB# text. This is configured here to be between 3 and 6 characters. We also want to reuse this ID in the value, so we capture it as a named subexpression called `id`. @@ -137,7 +137,7 @@ In the value we can reuse these named subexpression like this: ${pre}[AB#${id}](https://dev.azure.com/[your organization]/_workitems/edit/${id}) ``` -We start with the `pre` value, after which we build a markdown link with AB# combined with the `id` as the text and the `id` as parameter for the URL. We reference an Azure Board work item here. Of course you need to replace the `[your organization]` with the proper value for your ADO environment here. +We start with the `pre` value, after which we build a markdown link with AB# combined with the `id` as the text and the `id` as parameter for the URL. We reference an Azure Board work item here. Of course you need to replace the `[your organization]` with the proper value for your ADO environment here. With the examples above the text *AB#1234* would be translated to *[AB#1234(https://dev.azure.com/[your organization]/_workitems/edit/1234)*. ### `Content` @@ -153,5 +153,3 @@ The content is defined with these properties: | `urlReplacements` | A collection of [`Replacement`](#replacement) objects to use for URL paths in this content set, overruling any global setting. These replacements are applied to calculated destination paths for files in the content sets. This can be used to modify the path. The generated template removes /docs/ from paths and replaces it by a /. More information can be found under [`Replacement`](#replacement). | | `contentReplacements` | A collection of [`Replacement`](#replacement) objects to use for content of files in this content set, overruling any global setting. These replacements are applied to all content of markdown files in the content sets. This can be used to modify for instance URLs or other content items. More information can be found under [`Replacement`](#replacement). | | `externalFilePrefix` | The prefix to use for all referenced files in this content sets that are not part of the complete documentation set, like source files. It overrides any global prefix. This prefix is used in combination with the sub-path from the working folder. | - - From 0e554d4264850a905b08a89a2935351d6ccf4b79 Mon Sep 17 00:00:00 2001 From: Martin Tirion Date: Mon, 16 Dec 2024 16:22:00 +0100 Subject: [PATCH 12/12] Changed to centralized protocol list --- .../FileService/FileInfoService.cs | 7 +--- .../DocAssembler/FileService/Hyperlink.cs | 38 +++++++++++-------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs index 6bd8910..ecf6f99 100644 --- a/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs +++ b/src/DocAssembler/DocAssembler/FileService/FileInfoService.cs @@ -54,12 +54,7 @@ public List GetLocalHyperlinks(string root, string filePath) .Descendants() .Where(x => !x.UrlHasPointyBrackets && !string.IsNullOrEmpty(x.Url) && - !x.Url.StartsWith("https://", StringComparison.InvariantCulture) && - !x.Url.StartsWith("http://", StringComparison.InvariantCulture) && - !x.Url.StartsWith("ftps://", StringComparison.InvariantCulture) && - !x.Url.StartsWith("ftp://", StringComparison.InvariantCulture) && - !x.Url.StartsWith("mailto:", StringComparison.InvariantCulture) && - !x.Url.StartsWith("xref:", StringComparison.InvariantCulture)) + !Hyperlink.Protocols.Any(p => x.Url.StartsWith(p.Key, StringComparison.OrdinalIgnoreCase))) .Select(d => new Hyperlink(markdownFilePath, d.Line + 1, d.Column + 1, d.Url ?? string.Empty) { UrlSpanStart = d.UrlSpan.Start, diff --git a/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs b/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs index ac43374..20cbe45 100644 --- a/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs +++ b/src/DocAssembler/DocAssembler/FileService/Hyperlink.cs @@ -2,7 +2,9 @@ // Copyright (c) DocFx Companion Tools. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using System; using System.Text; +using DocAssembler.FileService; namespace DocAssembler.FileService; @@ -11,6 +13,19 @@ namespace DocAssembler.FileService; /// public class Hyperlink { + /// + /// Gets the protocol mappings to s. + /// + public static readonly Dictionary Protocols = new Dictionary() + { + { "https://", HyperlinkType.Webpage }, + { "http://", HyperlinkType.Webpage }, + { "ftps://", HyperlinkType.Ftp }, + { "ftp://", HyperlinkType.Ftp }, + { "mailto://", HyperlinkType.Mail }, + { "xref://", HyperlinkType.CrossReference }, + }; + private static readonly char[] _uriFragmentOrQueryString = new char[] { '#', '?' }; private static readonly char[] _additionalInvalidChars = @"\/?:*".ToArray(); private static readonly char[] _invalidPathChars = Path.GetInvalidPathChars().Concat(_additionalInvalidChars).ToArray(); @@ -41,23 +56,16 @@ public Hyperlink(string filePath, int line, int col, string url) LinkType = HyperlinkType.Empty; if (!string.IsNullOrWhiteSpace(url)) { - if (url.StartsWith("https://", StringComparison.InvariantCulture) || url.StartsWith("http://", StringComparison.InvariantCulture)) + foreach (var protocol in Protocols) { - LinkType = HyperlinkType.Webpage; - } - else if (url.StartsWith("ftps://", StringComparison.InvariantCulture) || url.StartsWith("ftp://", StringComparison.InvariantCulture)) - { - LinkType = HyperlinkType.Ftp; - } - else if (url.StartsWith("mailto:", StringComparison.InvariantCulture)) - { - LinkType = HyperlinkType.Mail; - } - else if (url.StartsWith("xref:", StringComparison.InvariantCulture)) - { - LinkType = HyperlinkType.CrossReference; + if (url.StartsWith(protocol.Key, StringComparison.OrdinalIgnoreCase)) + { + LinkType = protocol.Value; + break; + } } - else + + if (LinkType == HyperlinkType.Empty) { Url = UrlDecode(Url).NormalizePath();