Update navigation.yml (#780)

* add code comments with context

* address feedback from @lcawl @bmorelli25 in elastic/docs-projects#464

* add release notes config from @KOTungseth

* Revert "add release notes config from @KOTungseth"

This reverts commit 417f7af.

* net -> dotnet

* update navigation.yml and assembler.yml

* clean up

* more clean up

* Refactor navigation handling and introduce phantom support

Simplified navigation logic by restructuring method signatures and refining table of contents processing. Added support for phantom definitions in navigation files, ensuring appropriate handling of unlinked TOC folders. Refactored related classes to align with the new navigation paradigm.

---------

Co-authored-by: Martijn Laarman <[email protected]>

Author: colleenmcginnis

src/docs-assembler/AssembleSources.cs

@@ -33,6 +33,7 @@ public record TocConfigurationMapping
 
 public class AssembleSources
 {
+	public AssembleContext AssembleContext { get; }
 	public FrozenDictionary<string, AssemblerDocumentationSet> AssembleSets { get; }
 
 	public FrozenDictionary<Uri, TocTopLevelMapping> TocTopLevelMappings { get; }
@@ -51,16 +52,17 @@ public static async Task<AssembleSources> AssembleAsync(AssembleContext context,
 		return sources;
 	}
 
-	private AssembleSources(AssembleContext context, Checkout[] checkouts)
+	private AssembleSources(AssembleContext assembleContext, Checkout[] checkouts)
 	{
-		TocTopLevelMappings = GetConfiguredSources(context);
+		AssembleContext = assembleContext;
+		TocTopLevelMappings = GetConfiguredSources(assembleContext);
 
-		var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, context.Configuration);
-		UriResolver = new PublishEnvironmentUriResolver(TocTopLevelMappings, context.Environment);
+		var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, assembleContext.Configuration);
+		UriResolver = new PublishEnvironmentUriResolver(TocTopLevelMappings, assembleContext.Environment);
 		var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, UriResolver);
 		AssembleSets = checkouts
 			.Where(c => !c.Repository.Skip)
-			.Select(c => new AssemblerDocumentationSet(NullLoggerFactory.Instance, context, c, crossLinkResolver, TreeCollector))
+			.Select(c => new AssemblerDocumentationSet(NullLoggerFactory.Instance, assembleContext, c, crossLinkResolver, TreeCollector))
 			.ToDictionary(s => s.Checkout.Repository.Name, s => s)
 			.ToFrozenDictionary();
 
@@ -85,7 +87,7 @@ private AssembleSources(AssembleContext context, Checkout[] checkouts)
 				var file = tocFiles.FirstOrDefault(f => f.Exists);
 				if (file is null)
 				{
-					context.Collector.EmitWarning(context.ConfigurationPath.FullName, $"Unable to find toc file in {tocDirectory}");
+					assembleContext.Collector.EmitWarning(assembleContext.ConfigurationPath.FullName, $"Unable to find toc file in {tocDirectory}");
 					file = tocFiles.First();
 				}
 

src/docs-assembler/Cli/RepositoryCommands.cs

@@ -88,8 +88,8 @@ public async Task<int> BuildAll(
 
 		var navigation = new GlobalNavigation(assembleSources, navigationFile);
 
-		var pathProvider = new GlobalNavigationPathProvider(assembleSources, assembleContext);
-		var htmlWriter = new GlobalNavigationHtmlWriter(assembleContext, navigation, assembleSources);
+		var pathProvider = new GlobalNavigationPathProvider(navigationFile, assembleSources, assembleContext);
+		var htmlWriter = new GlobalNavigationHtmlWriter(navigationFile, assembleContext, navigation, assembleSources);
 
 		var builder = new AssemblerBuilder(logger, assembleContext, htmlWriter, pathProvider);
 		await builder.BuildAllAsync(assembleSources.AssembleSets, ctx);

src/docs-assembler/Navigation/GlobalNavigationFile.cs

@@ -3,25 +3,31 @@
 // See the LICENSE file in the project root for more information
 
 using System.Collections.Frozen;
+using System.IO.Abstractions;
 using Documentation.Assembler.Configuration;
 using Elastic.Markdown.IO.Configuration;
 using Elastic.Markdown.IO.Navigation;
 using YamlDotNet.RepresentationModel;
 
 namespace Documentation.Assembler.Navigation;
 
-public record GlobalNavigationFile
+public record GlobalNavigationFile : ITableOfContentsScope
 {
 	private readonly AssembleContext _context;
 	private readonly AssembleSources _assembleSources;
 
 	public IReadOnlyCollection<TocReference> TableOfContents { get; }
+	public IReadOnlyCollection<TocReference> Phantoms { get; }
+
+	public IDirectoryInfo ScopeDirectory { get; }
 
 	public GlobalNavigationFile(AssembleContext context, AssembleSources assembleSources)
 	{
 		_context = context;
 		_assembleSources = assembleSources;
-		TableOfContents = Deserialize();
+		TableOfContents = Deserialize("toc");
+		Phantoms = Deserialize("phantoms");
+		ScopeDirectory = _context.NavigationPath.Directory!;
 	}
 
 	public void EmitWarning(string message) =>
@@ -31,18 +37,15 @@ public void EmitError(string message) =>
 		_context.Collector.EmitWarning(_context.NavigationPath.FullName, message);
 
 
-	private IReadOnlyCollection<TocReference> Deserialize()
+	private IReadOnlyCollection<TocReference> Deserialize(string key)
 	{
 		var reader = new YamlStreamReader(_context.NavigationPath, _context.Collector);
 		try
 		{
 			foreach (var entry in reader.Read())
 			{
-				switch (entry.Key)
-				{
-					case "toc":
-						return ReadChildren(reader, entry.Entry, null, 0);
-				}
+				if (entry.Key == key)
+					return ReadChildren(key, reader, entry.Entry, null, 0);
 			}
 		}
 		catch (Exception e)
@@ -54,7 +57,8 @@ private IReadOnlyCollection<TocReference> Deserialize()
 		return [];
 	}
 
-	private IReadOnlyCollection<TocReference> ReadChildren(YamlStreamReader reader, KeyValuePair<YamlNode, YamlNode> entry, string? parent, int depth)
+	private IReadOnlyCollection<TocReference> ReadChildren(string key, YamlStreamReader reader, KeyValuePair<YamlNode, YamlNode> entry, string? parent,
+		int depth)
 	{
 		var entries = new List<TocReference>();
 		if (entry.Key is not YamlScalarNode { Value: not null } scalarKey)
@@ -71,15 +75,39 @@ private IReadOnlyCollection<TocReference> ReadChildren(YamlStreamReader reader,
 
 		foreach (var tocEntry in sequence.Children.OfType<YamlMappingNode>())
 		{
-			var child = ReadChild(reader, tocEntry, parent, depth);
+
+			var child =
+				key == "toc"
+					? ReadTocDefinition(reader, tocEntry, parent, depth)
+					: ReadPhantomDefinition(reader, tocEntry);
 			if (child is not null)
 				entries.Add(child);
 		}
 
 		return entries;
 	}
 
-	private TocReference? ReadChild(YamlStreamReader reader, YamlMappingNode tocEntry, string? parent, int depth)
+	private TocReference? ReadPhantomDefinition(YamlStreamReader reader, YamlMappingNode tocEntry)
+	{
+		foreach (var entry in tocEntry.Children)
+		{
+			var key = ((YamlScalarNode)entry.Key).Value;
+			switch (key)
+			{
+				case "toc":
+					var source = reader.ReadString(entry);
+					if (source != null && !source.Contains("://"))
+						source = ContentSourceMoniker.CreateString(NarrativeRepository.RepositoryName, source);
+					var sourceUri = new Uri(source!);
+					var tocReference = new TocReference(sourceUri, this, "", true, []);
+					return tocReference;
+			}
+		}
+
+		return null;
+	}
+
+	private TocReference? ReadTocDefinition(YamlStreamReader reader, YamlMappingNode tocEntry, string? parent, int depth)
 	{
 		string? repository = null;
 		string? source = null;
@@ -139,9 +167,6 @@ private IReadOnlyCollection<TocReference> ReadChildren(YamlStreamReader reader,
 		}
 
 		var navigationItems = new List<ITocItem>();
-		//TODO not needed
-		//var tocChildren = mapping.TableOfContentsConfiguration.TableOfContents;
-		//navigationItems.AddRange(tocChildren);
 
 		foreach (var entry in tocEntry.Children)
 		{
@@ -155,7 +180,7 @@ private IReadOnlyCollection<TocReference> ReadChildren(YamlStreamReader reader,
 						continue;
 					}
 
-					var children = ReadChildren(reader, entry, parent, depth + 1);
+					var children = ReadChildren("toc", reader, entry, parent, depth + 1);
 					navigationItems.AddRange(children);
 					break;
 			}
@@ -166,4 +191,5 @@ private IReadOnlyCollection<TocReference> ReadChildren(YamlStreamReader reader,
 		var tocReference = new TocReference(sourceUri, mapping.TableOfContentsConfiguration, path, true, navigationItems);
 		return tocReference;
 	}
+
 }

src/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs

@@ -3,53 +3,50 @@
 // See the LICENSE file in the project root for more information
 
 using System.Collections.Concurrent;
+using System.Collections.Immutable;
 using Elastic.Markdown.IO.Configuration;
 using Elastic.Markdown.IO.Navigation;
 using Elastic.Markdown.Slices;
 
 namespace Documentation.Assembler.Navigation;
 
-public class GlobalNavigationHtmlWriter(AssembleContext assembleContext, GlobalNavigation navigation, AssembleSources assembleSources) : INavigationHtmlWriter
+public class GlobalNavigationHtmlWriter(
+	GlobalNavigationFile navigationFile,
+	AssembleContext assembleContext,
+	GlobalNavigation globalNavigation,
+	AssembleSources assembleSources) : INavigationHtmlWriter
 {
 	private readonly AssembleContext _assembleContext = assembleContext;
 	private readonly ConcurrentDictionary<Uri, string> _renderedNavigationCache = [];
 
-	private (DocumentationGroup, Uri) GetRealNavigationRoot(INavigation navigation)
-	{
-		if (navigation is not DocumentationGroup group)
-			throw new InvalidOperationException($"Expected a {nameof(DocumentationGroup)}");
+	private ImmutableHashSet<Uri> Phantoms { get; } = [.. navigationFile.Phantoms.Select(p => p.Source)];
 
-
-		if (group.NavigationRoot is TableOfContentsTree tree)
+	private (DocumentationGroup, Uri) GetRealNavigationRoot(TableOfContentsTree tree)
+	{
+		if (!assembleSources.TocTopLevelMappings.TryGetValue(tree.Source, out var topLevelUri))
 		{
-			if (!assembleSources.TocTopLevelMappings.TryGetValue(tree.Source, out var topLevelUri))
-			{
-				_assembleContext.Collector.EmitWarning(_assembleContext.NavigationPath.FullName, $"Could not find a top level mapping for {tree.Source}");
-				return (tree, tree.Source);
-			}
-
-			if (!assembleSources.TreeCollector.TryGetTableOfContentsTree(topLevelUri.TopLevelSource, out var topLevel))
-			{
-				_assembleContext.Collector.EmitWarning(_assembleContext.NavigationPath.FullName, $"Could not find a toc tree for {topLevelUri.TopLevelSource}");
-				return (tree, tree.Source);
-
-			}
-			return (topLevel, topLevelUri.TopLevelSource);
-
+			_assembleContext.Collector.EmitWarning(_assembleContext.NavigationPath.FullName, $"Could not find a top level mapping for {tree.Source}");
+			return (tree, tree.Source);
 		}
-		else if (group.NavigationRoot is DocumentationGroup)
+
+		if (!assembleSources.TreeCollector.TryGetTableOfContentsTree(topLevelUri.TopLevelSource, out var topLevel))
 		{
-			var source = group.FolderName == "reference/index.md"
-				? new Uri("docs-content://reference/")
-				: throw new InvalidOperationException($"{group.FolderName} is not a valid navigation root");
-			return (group, source);
+			_assembleContext.Collector.EmitWarning(_assembleContext.NavigationPath.FullName, $"Could not find a toc tree for {topLevelUri.TopLevelSource}");
+			return (tree, tree.Source);
+
 		}
-		throw new InvalidOperationException($"Unknown navigation root {group.NavigationRoot}");
+		return (topLevel, topLevelUri.TopLevelSource);
 	}
 
 	public async Task<string> RenderNavigation(INavigation currentRootNavigation, Cancel ctx = default)
 	{
-		var (navigation, source) = GetRealNavigationRoot(currentRootNavigation);
+		if (currentRootNavigation is not TableOfContentsTree tree)
+			throw new InvalidOperationException($"Expected a {nameof(DocumentationGroup)}");
+
+		if (Phantoms.Contains(tree.Source))
+			return string.Empty;
+
+		var (navigation, source) = GetRealNavigationRoot(tree);
 		if (_renderedNavigationCache.TryGetValue(source, out var value))
 			return value;
 
@@ -64,17 +61,13 @@ public async Task<string> RenderNavigation(INavigation currentRootNavigation, Ca
 		var model = CreateNavigationModel(navigation);
 		value = await ((INavigationHtmlWriter)this).Render(model, ctx);
 		_renderedNavigationCache[source] = value;
-		if (source == new Uri("docs-content://extend"))
-		{
-		}
-
 
 		return value;
 	}
 
 	private NavigationViewModel CreateNavigationModel(DocumentationGroup group)
 	{
-		var topLevelItems = navigation.TopLevelItems;
+		var topLevelItems = globalNavigation.TopLevelItems;
 		return new NavigationViewModel
 		{
 			Title = group.Index?.NavigationTitle ?? "Docs",

src/docs-assembler/Navigation/GlobalNavigationPathProvider.cs

@@ -2,20 +2,12 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
-using System.Collections.Frozen;
 using System.Collections.Immutable;
-using System.Diagnostics.CodeAnalysis;
 using System.IO.Abstractions;
-using Documentation.Assembler.Building;
-using Documentation.Assembler.Configuration;
-using Documentation.Assembler.Sourcing;
 using Elastic.Markdown;
-using Elastic.Markdown.CrossLinks;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.Configuration;
-using Elastic.Markdown.IO.Navigation;
-using Microsoft.Extensions.Logging.Abstractions;
 
 namespace Documentation.Assembler.Navigation;
 
@@ -25,8 +17,9 @@ public record GlobalNavigationPathProvider : IDocumentationFileOutputProvider
 	private readonly AssembleContext _context;
 
 	private ImmutableSortedSet<string> TableOfContentsPrefixes { get; }
+	private ImmutableSortedSet<string> PhantomPrefixes { get; }
 
-	public GlobalNavigationPathProvider(AssembleSources assembleSources, AssembleContext context)
+	public GlobalNavigationPathProvider(GlobalNavigationFile navigationFile, AssembleSources assembleSources, AssembleContext context)
 	{
 		_assembleSources = assembleSources;
 		_context = context;
@@ -36,45 +29,14 @@ public GlobalNavigationPathProvider(AssembleSources assembleSources, AssembleCon
 			.Select(v => v.Source.ToString())
 			.OrderByDescending(v => v.Length)
 		];
-	}
-
-	/*
-	public IFileInfo? LocateDocSetYaml(Uri crossLinkUri)
-	{
-		if (!TryGetCheckout(crossLinkUri, out var checkout))
-			return null;
-
-		var tocDirectory = _readFs.DirectoryInfo.New(Path.Combine(checkout.Directory.FullName, crossLinkUri.Host, crossLinkUri.AbsolutePath.TrimStart('/')));
-		if (!tocDirectory.Exists)
-		{
-			_context.Collector.EmitError(_context.NavigationPath, $"Unable to find toc directory: {tocDirectory.FullName}");
-			return null;
-		}
-
-		var docsetYaml = _readFs.FileInfo.New(Path.Combine(tocDirectory.FullName, "docset.yml"));
-		var tocYaml = _readFs.FileInfo.New(Path.Combine(tocDirectory.FullName, "toc.yml"));
-		if (!docsetYaml.Exists && !tocYaml.Exists)
-		{
-			_context.Collector.EmitError(_context.NavigationPath, $"Unable to find docset.yml or toc.yml in: {tocDirectory.FullName}");
-			return null;
-		}
 
-		return docsetYaml.Exists ? docsetYaml : tocYaml;
+		PhantomPrefixes = [..navigationFile.Phantoms
+			.Select(p => p.Source.ToString())
+			.OrderByDescending(v => v.Length)
+			.ToArray()
+		];
 	}
 
-	public bool TryGetCheckout(Uri crossLinkUri, [NotNullWhen(true)] out Checkout? checkout)
-	{
-		if (_checkoutsLookup.TryGetValue(crossLinkUri.Scheme, out checkout))
-			return true;
-
-		_context.Collector.EmitError(_context.ConfigurationPath,
-			!_repoConfigLookup.TryGetValue(crossLinkUri.Scheme, out _)
-				? $"Repository: '{crossLinkUri.Scheme}' is not defined in assembler.yml"
-				: $"Unable to find checkout for repository: {crossLinkUri.Scheme}"
-		);
-		return false;
-	}*/
-
 	public IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath)
 	{
 		if (relativePath.StartsWith("_static/", StringComparison.Ordinal))
@@ -124,6 +86,12 @@ public bool TryGetCheckout(Uri crossLinkUri, [NotNullWhen(true)] out Checkout? c
 			if (relativePath.EndsWith(".asciidoc", StringComparison.Ordinal))
 				return null;
 
+			foreach (var prefix in PhantomPrefixes)
+			{
+				if (lookup.StartsWith(prefix, StringComparison.Ordinal))
+					return null;
+			}
+
 			var fallBack = fs.Path.Combine(outputDirectory.FullName, "_failed", repositoryName, relativePath);
 			_context.Collector.EmitError(_context.NavigationPath, $"No toc for output path: '{lookup}' falling back to: '{fallBack}'");
 			return fs.FileInfo.New(fallBack);