feature/assembler global toc (#770)

* Add global navigation and improve diagnostic handling

This commit introduces a GlobalNavigation feature, parsing the new `navigation.yml` file to handle global navigation references. It also replaces `BuildContext` with `DiagnosticsCollector` for diagnostics, ensuring clearer separation of concerns. Additionally, it includes new tests and updates project files for better testing and navigation handling.

* Use navigation.yml to control url path prefixes for resolving

* path_prefix is mandatory except for top level docs-content references

* stage commit

* stage commit

* File copy now in a decent place

* temporary home for in flux paths

* fix failing test

* remove not referenced test project

* Restrict docset.yml configs that define toc.yml sections to ONLY link to sub toc.yml files (#767)

* Restrict docset.yml configs that define toc.yml sections to ONLY link to sub toc.yml files

* relax check for narrative docs too, plays by different rules

* dotnet format

* reference Project directly

* Fix layout and horizontal scrollable tables (#772)

* Fix layout and horizontal scrollable tables

mend

* Use spacing var

* Add custom scrollbar

* tweaks

* Add table of contents scope management and refactor diagnostics (#774)

* Add table of contents scope management and refactor diagnostics

Introduced `ITableOfContentsScope` to manage scope boundaries, improving link validation and table of contents processing. Updated diagnostics to emit hints for images outside the ToC scope and refactored functions for better reusability. Adjusted related tests and internal structures accordingly.

* Add TODO to make this an error

* post merge build fixes

* good enough state

---------

Co-authored-by: Jan Calanog <[email protected]>

Author: Mpdreamz

docs-builder.sln

@@ -73,6 +73,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "update-reference-index", "u
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-lambda-index-publisher", "src\docs-lambda-index-publisher\docs-lambda-index-publisher.csproj", "{C559D52D-100B-4B2B-BE87-2344D835761D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-assembler.Tests", "tests\docs-assembler.Tests\src\docs-assembler.Tests\docs-assembler.Tests.csproj", "{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -122,6 +124,10 @@ Global
 		{C559D52D-100B-4B2B-BE87-2344D835761D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -138,5 +144,6 @@ Global
 		{6554F917-73CE-4B3D-9101-F28EAA762C6B} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{9FEC15F6-13F8-40B1-A66A-EB054E49E680} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{C559D52D-100B-4B2B-BE87-2344D835761D} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5}
 	EndGlobalSection
 EndGlobal

src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs

@@ -24,7 +24,7 @@ protected override void HandleItem(Diagnostic diagnostic)
 			_errors.Add(diagnostic);
 		else if (diagnostic.Severity == Severity.Warning)
 			_warnings.Add(diagnostic);
-		else
+		else if (!NoHints)
 			_hints.Add(diagnostic);
 	}
 

src/Elastic.Markdown/BuildContext.cs

@@ -29,6 +29,8 @@ public record BuildContext
 
 	public bool Force { get; init; }
 
+	public bool SkipMetadata { get; init; }
+
 	// This property is used to determine if the site should be indexed by search engines
 	public bool AllowIndexing { get; init; }
 
@@ -39,15 +41,6 @@ public string? UrlPathPrefix
 		init => _urlPathPrefix = value;
 	}
 
-	private readonly string? _staticUrlPathPrefix;
-	public string? StaticUrlPathPrefix
-	{
-		get => !string.IsNullOrWhiteSpace(_staticUrlPathPrefix)
-			? $"/{_staticUrlPathPrefix.Trim('/')}"
-			: UrlPathPrefix;
-		init => _staticUrlPathPrefix = value;
-	}
-
 	public BuildContext(IFileSystem fileSystem)
 		: this(new DiagnosticsCollector([]), fileSystem, fileSystem, null, null) { }
 

src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs

@@ -45,7 +45,7 @@ public interface ICrossLinkResolver
 public class CrossLinkResolver(CrossLinkFetcher fetcher, IUriEnvironmentResolver? uriResolver = null) : ICrossLinkResolver
 {
 	private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty;
-	private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new PreviewEnvironmentUriResolver();
+	private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new IsolatedBuildEnvironmentUriResolver();
 
 	public async Task<FetchedCrossLinks> FetchLinks()
 	{

src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs

@@ -9,7 +9,7 @@ public interface IUriEnvironmentResolver
 	Uri Resolve(Uri crossLinkUri, string path);
 }
 
-public class PreviewEnvironmentUriResolver : IUriEnvironmentResolver
+public class IsolatedBuildEnvironmentUriResolver : IUriEnvironmentResolver
 {
 	private static Uri BaseUri { get; } = new("https://docs-v3-preview.elastic.dev");
 

src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs

@@ -85,10 +85,12 @@ public class DiagnosticsCollector(IReadOnlyCollection<IDiagnosticsOutput> output
 
 	public HashSet<string> OffendingFiles { get; } = [];
 
-	public HashSet<string> InUseSubstitutionKeys { get; } = [];
+	public ConcurrentDictionary<string, bool> InUseSubstitutionKeys { get; } = [];
 
 	public ConcurrentBag<string> CrossLinks { get; } = [];
 
+	public bool NoHints { get; init; }
+
 	public Task StartAsync(Cancel cancellationToken)
 	{
 		if (_started is not null)
@@ -117,6 +119,8 @@ void Drain()
 		{
 			while (Channel.Reader.TryRead(out var item))
 			{
+				if (item.Severity == Severity.Hint && NoHints)
+					continue;
 				IncrementSeverityCount(item);
 				HandleItem(item);
 				_ = OffendingFiles.Add(item.File);
@@ -132,7 +136,7 @@ private void IncrementSeverityCount(Diagnostic item)
 			_ = Interlocked.Increment(ref _errors);
 		else if (item.Severity == Severity.Warning)
 			_ = Interlocked.Increment(ref _warnings);
-		else if (item.Severity == Severity.Hint)
+		else if (item.Severity == Severity.Hint && !NoHints)
 			_ = Interlocked.Increment(ref _hints);
 	}
 
@@ -175,5 +179,5 @@ public async ValueTask DisposeAsync()
 	}
 
 	public void CollectUsedSubstitutionKey(ReadOnlySpan<char> key) =>
-		_ = InUseSubstitutionKeys.Add(key.ToString());
+		_ = InUseSubstitutionKeys.TryAdd(key.ToString(), true);
 }

src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs

@@ -98,6 +98,28 @@ public static void EmitWarning(this BuildContext context, IFileInfo file, string
 		context.Collector.Channel.Write(d);
 	}
 
+	public static void EmitError(this DiagnosticsCollector collector, IFileInfo file, string message, Exception? e = null)
+	{
+		var d = new Diagnostic
+		{
+			Severity = Severity.Error,
+			File = file.FullName,
+			Message = message + (e != null ? Environment.NewLine + e : string.Empty),
+		};
+		collector.Channel.Write(d);
+	}
+
+	public static void EmitWarning(this DiagnosticsCollector collector, IFileInfo file, string message)
+	{
+		var d = new Diagnostic
+		{
+			Severity = Severity.Warning,
+			File = file.FullName,
+			Message = message,
+		};
+		collector.Channel.Write(d);
+	}
+
 	public static void EmitError(this IBlockExtension block, string message, Exception? e = null)
 	{
 		if (block.SkipValidation)

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -20,8 +20,14 @@ public interface IConversionCollector
 	void Collect(MarkdownFile file, MarkdownDocument document, string html);
 }
 
+public interface IDocumentationFileOutputProvider
+{
+	IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath);
+}
+
 public class DocumentationGenerator
 {
+	private readonly IDocumentationFileOutputProvider? _documentationFileOutputProvider;
 	private readonly ILogger _logger;
 	private readonly IFileSystem _writeFileSystem;
 	private readonly IDocumentationFileExporter _documentationFileExporter;
@@ -34,10 +40,12 @@ public class DocumentationGenerator
 	public DocumentationGenerator(
 		DocumentationSet docSet,
 		ILoggerFactory logger,
+		IDocumentationFileOutputProvider? documentationFileOutputProvider = null,
 		IDocumentationFileExporter? documentationExporter = null,
 		IConversionCollector? conversionCollector = null
 	)
 	{
+		_documentationFileOutputProvider = documentationFileOutputProvider;
 		_writeFileSystem = docSet.Build.WriteFileSystem;
 		_logger = logger.CreateLogger(nameof(DocumentationGenerator));
 
@@ -74,7 +82,7 @@ public async Task ResolveDirectoryTree(Cancel ctx)
 	public async Task GenerateAll(Cancel ctx)
 	{
 		var generationState = GetPreviousGenerationState();
-		if (Context.Force || generationState == null)
+		if (!Context.SkipMetadata && (Context.Force || generationState == null))
 			DocumentationSet.ClearOutputDirectory();
 
 		if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
@@ -91,17 +99,22 @@ public async Task GenerateAll(Cancel ctx)
 
 		await ExtractEmbeddedStaticResources(ctx);
 
-		_logger.LogInformation($"Completing diagnostics channel");
-		Context.Collector.Channel.TryComplete();
+		if (Context.SkipMetadata)
+			return;
 
 		_logger.LogInformation($"Generating documentation compilation state");
 		await GenerateDocumentationState(ctx);
 
 		_logger.LogInformation($"Generating links.json");
 		await GenerateLinkReference(ctx);
+	}
 
+	public async Task StopDiagnosticCollection(Cancel ctx)
+	{
 		_logger.LogInformation($"Completing diagnostics channel");
+		Context.Collector.Channel.TryComplete();
 
+		_logger.LogInformation($"Stopping diagnostics collector");
 		await Context.Collector.StopAsync(ctx);
 
 		_logger.LogInformation($"Completed diagnostics channel");
@@ -140,7 +153,8 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 	private void HintUnusedSubstitutionKeys()
 	{
 		var definedKeys = new HashSet<string>(Context.Configuration.Substitutions.Keys.ToArray());
-		var keysNotInUse = definedKeys.Except(Context.Collector.InUseSubstitutionKeys).ToArray();
+		var inUse = new HashSet<string>(Context.Collector.InUseSubstitutionKeys.Keys);
+		var keysNotInUse = definedKeys.Except(inUse).ToArray();
 		// If we have less than 20 unused keys emit them separately
 		// Otherwise emit one hint with all of them for brevity
 		if (keysNotInUse.Length >= 20)
@@ -170,6 +184,8 @@ private async Task ExtractEmbeddedStaticResources(Cancel ctx)
 			var path = a.Replace("Elastic.Markdown.", "").Replace("_static.", $"_static{Path.DirectorySeparatorChar}");
 
 			var outputFile = OutputFile(path);
+			if (outputFile is null)
+				continue;
 			await _documentationFileExporter.CopyEmbeddedResource(outputFile, resourceStream, ctx);
 			_logger.LogDebug("Copied static embedded resource {Path}", path);
 		}
@@ -186,14 +202,21 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 		}
 
 		_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
+		//TODO send file to OutputFile() so we can validate its scope is defined in navigation.yml
 		var outputFile = OutputFile(file.RelativePath);
-		await _documentationFileExporter.ProcessFile(file, outputFile, token);
+		if (outputFile is not null)
+			await _documentationFileExporter.ProcessFile(file, outputFile, token);
 	}
 
-	private IFileInfo OutputFile(string relativePath)
+	private IFileInfo? OutputFile(string relativePath)
 	{
 		var outputFile = _writeFileSystem.FileInfo.New(Path.Combine(DocumentationSet.OutputDirectory.FullName, relativePath));
-		return outputFile;
+		if (relativePath.StartsWith("_static"))
+			return outputFile;
+
+		return _documentationFileOutputProvider is not null
+			? _documentationFileOutputProvider.OutputFile(DocumentationSet, outputFile, relativePath)
+			: outputFile;
 	}
 
 	private bool CompilationNotNeeded(GenerationState? generationState, out HashSet<string> offendingFiles,

src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs

@@ -74,7 +74,7 @@ public ConfigurationFile(BuildContext context)
 		var redirectFile = new RedirectFile(redirectFileInfo, _context);
 		Redirects = redirectFile.Redirects;
 
-		var reader = new YamlStreamReader(sourceFile, _context);
+		var reader = new YamlStreamReader(sourceFile, _context.Collector);
 		try
 		{
 			foreach (var entry in reader.Read())
@@ -120,7 +120,7 @@ public ConfigurationFile(BuildContext context)
 			}
 
 			//we read it twice to ensure we read 'toc' last
-			reader = new YamlStreamReader(sourceFile, _context);
+			reader = new YamlStreamReader(sourceFile, _context.Collector);
 			foreach (var entry in reader.Read())
 			{
 				switch (entry.Key)

src/Elastic.Markdown/IO/Configuration/RedirectFile.cs

@@ -22,7 +22,7 @@ public RedirectFile(IFileInfo source, BuildContext context)
 		if (!source.Exists)
 			return;
 
-		var reader = new YamlStreamReader(Source, Context);
+		var reader = new YamlStreamReader(Source, Context.Collector);
 		try
 		{
 			foreach (var entry in reader.Read())

src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs

@@ -239,7 +239,7 @@ public IReadOnlyCollection<ITocItem> ReadChildren(YamlStreamReader reader, KeyVa
 		if (!found)
 			return null;
 
-		var tocYamlReader = new YamlStreamReader(source, _context);
+		var tocYamlReader = new YamlStreamReader(source, _context.Collector);
 		foreach (var kv in tocYamlReader.Read())
 		{
 			switch (kv.Key)

src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs

@@ -16,10 +16,10 @@ public record YamlToplevelKey
 	public required KeyValuePair<YamlNode, YamlNode> Entry { get; init; }
 }
 
-public class YamlStreamReader(IFileInfo source, BuildContext context)
+public class YamlStreamReader(IFileInfo source, DiagnosticsCollector collector)
 {
-	public IFileInfo Source { get; init; } = source;
-	public BuildContext Context { get; init; } = context;
+	private IFileInfo Source { get; init; } = source;
+	private DiagnosticsCollector Collector { get; init; } = collector;
 
 	public IEnumerable<YamlToplevelKey> Read()
 	{
@@ -30,7 +30,7 @@ public IEnumerable<YamlToplevelKey> Read()
 
 		if (yaml.Documents.Count == 0)
 		{
-			Context.EmitWarning(Source, "empty redirect file");
+			Collector.EmitWarning(Source, "empty redirect file");
 			yield break;
 		}
 		// Examine the stream
@@ -171,7 +171,7 @@ public void EmitWarning(string message, YamlNode? node) =>
 		EmitWarning(message, node?.Start, node?.End, (node as YamlScalarNode)?.Value?.Length);
 
 	public void EmitError(string message, Exception e) =>
-		Context.Collector.EmitError(Source.FullName, message, e);
+		Collector.EmitError(Source.FullName, message, e);
 
 	private void EmitError(string message, Mark? start = null, Mark? end = null, int? length = null)
 	{
@@ -185,7 +185,7 @@ private void EmitError(string message, Mark? start = null, Mark? end = null, int
 			Column = start.HasValue ? (int)start.Value.Column : null,
 			Length = length
 		};
-		Context.Collector.Channel.Write(d);
+		Collector.Channel.Write(d);
 	}
 	public void EmitWarning(string message, Mark? start = null, Mark? end = null, int? length = null)
 	{
@@ -199,6 +199,6 @@ public void EmitWarning(string message, Mark? start = null, Mark? end = null, in
 			Column = start.HasValue ? (int)start.Value.Column : null,
 			Length = length
 		};
-		Context.Collector.Channel.Write(d);
+		Collector.Channel.Write(d);
 	}
 }

src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs

@@ -46,21 +46,25 @@ public static GitCheckoutInformation Create(IDirectoryInfo source, IFileSystem f
 		}
 
 		var fakeRef = Guid.NewGuid().ToString()[..16];
-		var gitConfig = Git(source, ".git/config");
+		var gitConfig = Git(source, Path.Combine(".git", "config"));
 		if (!gitConfig.Exists)
 		{
-			logger?.LogInformation("Git checkout information not available.");
-			return Unavailable;
+			gitConfig = Git(source, Path.Combine("..", ".git", "config"));
+			if (!gitConfig.Exists)
+			{
+				logger?.LogInformation("Git checkout information not available.");
+				return Unavailable;
+			}
 		}
 
-		var head = Read(source, ".git/HEAD") ?? fakeRef;
+		var head = Read(source, Path.Combine(".git", "HEAD")) ?? fakeRef;
 		var gitRef = head;
 		var branch = head.Replace("refs/heads/", string.Empty);
 		//not detached HEAD
 		if (head.StartsWith("ref:"))
 		{
 			head = head.Replace("ref: ", string.Empty);
-			gitRef = Read(source, ".git/" + head) ?? fakeRef;
+			gitRef = Read(source, Path.Combine(".git", head)) ?? fakeRef;
 			branch = branch.Replace("ref: ", string.Empty);
 		}
 		else