File copy now in a decent place

Mpdreamz · Mpdreamz · commit 73ba35f55640 · 2025-03-18T22:38:35.000+01:00
diff --git a/src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs b/src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs
@@ -24,7 +24,7 @@ protected override void HandleItem(Diagnostic diagnostic)
 			_errors.Add(diagnostic);
 		else if (diagnostic.Severity == Severity.Warning)
 			_warnings.Add(diagnostic);
-		else
+		else if (!NoHints)
 			_hints.Add(diagnostic);
 	}
 
diff --git a/src/Elastic.Markdown/BuildContext.cs b/src/Elastic.Markdown/BuildContext.cs
@@ -29,6 +29,8 @@ public record BuildContext
 
 	public bool Force { get; init; }
 
+	public bool SkipMetadata { get; init; }
+
 	// This property is used to determine if the site should be indexed by search engines
 	public bool AllowIndexing { get; init; }
 
diff --git a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs
@@ -85,10 +85,12 @@ public class DiagnosticsCollector(IReadOnlyCollection<IDiagnosticsOutput> output
 
 	public HashSet<string> OffendingFiles { get; } = [];
 
-	public HashSet<string> InUseSubstitutionKeys { get; } = [];
+	public ConcurrentDictionary<string, bool> InUseSubstitutionKeys { get; } = [];
 
 	public ConcurrentBag<string> CrossLinks { get; } = [];
 
+	public bool NoHints { get; init; }
+
 	public Task StartAsync(Cancel cancellationToken)
 	{
 		if (_started is not null)
@@ -117,6 +119,8 @@ void Drain()
 		{
 			while (Channel.Reader.TryRead(out var item))
 			{
+				if (item.Severity == Severity.Hint && NoHints)
+					continue;
 				IncrementSeverityCount(item);
 				HandleItem(item);
 				_ = OffendingFiles.Add(item.File);
@@ -132,7 +136,7 @@ private void IncrementSeverityCount(Diagnostic item)
 			_ = Interlocked.Increment(ref _errors);
 		else if (item.Severity == Severity.Warning)
 			_ = Interlocked.Increment(ref _warnings);
-		else if (item.Severity == Severity.Hint)
+		else if (item.Severity == Severity.Hint && !NoHints)
 			_ = Interlocked.Increment(ref _hints);
 	}
 
@@ -175,5 +179,5 @@ public async ValueTask DisposeAsync()
 	}
 
 	public void CollectUsedSubstitutionKey(ReadOnlySpan<char> key) =>
-		_ = InUseSubstitutionKeys.Add(key.ToString());
+		_ = InUseSubstitutionKeys.TryAdd(key.ToString(), true);
 }
diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs
@@ -82,7 +82,7 @@ public async Task ResolveDirectoryTree(Cancel ctx)
 	public async Task GenerateAll(Cancel ctx)
 	{
 		var generationState = GetPreviousGenerationState();
-		if (Context.Force || generationState == null)
+		if (!Context.SkipMetadata && (Context.Force || generationState == null))
 			DocumentationSet.ClearOutputDirectory();
 
 		if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
@@ -99,6 +99,9 @@ public async Task GenerateAll(Cancel ctx)
 
 		await ExtractEmbeddedStaticResources(ctx);
 
+		if (Context.SkipMetadata)
+			return;
+
 		_logger.LogInformation($"Generating documentation compilation state");
 		await GenerateDocumentationState(ctx);
 
@@ -150,7 +153,8 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 	private void HintUnusedSubstitutionKeys()
 	{
 		var definedKeys = new HashSet<string>(Context.Configuration.Substitutions.Keys.ToArray());
-		var keysNotInUse = definedKeys.Except(Context.Collector.InUseSubstitutionKeys).ToArray();
+		var inUse = new HashSet<string>(Context.Collector.InUseSubstitutionKeys.Keys);
+		var keysNotInUse = definedKeys.Except(inUse).ToArray();
 		// If we have less than 20 unused keys emit them separately
 		// Otherwise emit one hint with all of them for brevity
 		if (keysNotInUse.Length >= 20)
diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs
@@ -91,7 +91,7 @@ public DocumentationSet(BuildContext build, ILoggerFactory logger, ICrossLinkRes
 			.SelectMany(extension => extension.ScanDocumentationFiles(ScanDocumentationFiles, DefaultFileHandling))
 			.ToArray();
 
-		Files = files.Concat(additionalSources).ToArray();
+		Files = files.Concat(additionalSources).Where(f => f is not ExcludedFile).ToArray();
 
 		LastWrite = Files.Max(f => f.SourceFile.LastWriteTimeUtc);
 
@@ -154,7 +154,7 @@ private DocumentationFile DefaultFileHandling(IFileInfo file, IDirectoryInfo sou
 			if (documentationFile is not null)
 				return documentationFile;
 		}
-		return new StaticFile(file, sourceDirectory);
+		return new ExcludedFile(file, sourceDirectory);
 	}
 
 	private void ValidateRedirectsExists()
diff --git a/src/docs-assembler/Building/AssemblerBuilder.cs b/src/docs-assembler/Building/AssemblerBuilder.cs
@@ -20,11 +20,15 @@ public async Task BuildAllAsync(IReadOnlyCollection<Checkout> checkouts, Publish
 		var uriResolver = new PublishEnvironmentUriResolver(globalNavigation, environment);
 		var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, uriResolver);
 
+		if (context.OutputDirectory.Exists)
+			context.OutputDirectory.Delete(true);
+		context.OutputDirectory.Create();
+
 		foreach (var checkout in checkouts)
 		{
 			if (checkout.Repository.Skip)
 			{
-				context.Collector.EmitWarning(context.ConfigurationPath.FullName, $"Skipping {checkout.Repository.Origin} as its not yet been migrated to V3");
+				context.Collector.EmitWarning(context.ConfigurationPath.FullName, $"Skipping {checkout.Repository.Origin} as its marked as skip in configuration");
 				continue;
 			}
 
@@ -55,8 +59,9 @@ private async Task BuildAsync(Checkout checkout, PublishEnvironment environment,
 		var buildContext = new BuildContext(context.Collector, context.ReadFileSystem, context.WriteFileSystem, path, output)
 		{
 			UrlPathPrefix = environment.PathPrefix,
-			Force = true,
-			AllowIndexing = environment.AllowIndexing
+			Force = false,
+			AllowIndexing = environment.AllowIndexing,
+			SkipMetadata = true
 		};
 
 		var set = new DocumentationSet(buildContext, logger, crossLinkResolver);
diff --git a/src/docs-assembler/Cli/RepositoryCommands.cs b/src/docs-assembler/Cli/RepositoryCommands.cs
@@ -63,7 +63,11 @@ public async Task<int> BuildAll(
 		var githubEnvironmentInput = githubActionsService.GetInput("environment");
 		environment ??= !string.IsNullOrEmpty(githubEnvironmentInput) ? githubEnvironmentInput : "dev";
 
-		await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService);
+		await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService)
+		{
+			NoHints = true
+		};
+
 		_ = collector.StartAsync(ctx);
 
 		var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null, null)
diff --git a/src/docs-assembler/Navigation/GlobalNavigation.cs b/src/docs-assembler/Navigation/GlobalNavigation.cs
@@ -70,7 +70,9 @@ public string GetSubPath(Uri crossLinkUri, ref string path)
 		}
 		if (match is null || !_tocLookup.TryGetValue(match, out var toc))
 		{
-			_context.Collector.EmitError(_context.NavigationPath, $"Unable to find defined toc for url: {crossLinkUri}");
+			//TODO remove
+			if (crossLinkUri.Scheme != "asciidocalypse")
+				_context.Collector.EmitError(_context.NavigationPath, $"Unable to find defined toc for url: {crossLinkUri}");
 			return $"reference/{crossLinkUri.Scheme}";
 		}
 		path = path.AsSpan().TrimStart(toc.SourcePrefix).ToString();
@@ -85,10 +87,22 @@ public string GetSubPath(Uri crossLinkUri, ref string path)
 
 		var outputDirectory = documentationSet.OutputDirectory;
 		var fs = defaultOutputFile.FileSystem;
-		var relativePathSpan = relativePath.AsSpan();
 
-		var lookup = $"{documentationSet.Name}://{relativePath}";
-		var match = TableOfContentsPrefixes.FirstOrDefault(prefix => lookup.StartsWith(prefix, StringComparison.Ordinal));
+		var lookup = $"{documentationSet.Name}://{relativePath}".AsSpan();
+		if (lookup.StartsWith("docs-content://serverless/", StringComparison.Ordinal))
+			return null;
+		if (lookup.StartsWith("eland://sphinx/", StringComparison.Ordinal))
+			return null;
+
+
+		string? match = null;
+		foreach (var prefix in TableOfContentsPrefixes)
+		{
+			if (!lookup.StartsWith(prefix, StringComparison.Ordinal))
+				continue;
+			match = prefix;
+			break;
+		}
 		if (match is null || !_tocLookup.TryGetValue(match, out var toc))
 		{
 			if (relativePath.StartsWith("raw-migrated-files/", StringComparison.Ordinal))
@@ -108,7 +122,12 @@ public string GetSubPath(Uri crossLinkUri, ref string path)
 			_context.Collector.EmitError(_context.NavigationPath, $"No toc for output path: '{lookup}' falling back to: '{fallBack}'");
 			return fs.FileInfo.New(fallBack);
 		}
-		var path = fs.Path.Combine(outputDirectory.FullName, toc.PathPrefix, relativePath);
+
+		var newPath = relativePath.AsSpan().TrimStart(toc.SourcePrefix.TrimEnd('/')).TrimStart('/').ToString();
+		var path = fs.Path.Combine(outputDirectory.FullName, toc.PathPrefix, newPath);
+		if (path.Contains("deploy-manage"))
+		{
+		}
 		return fs.FileInfo.New(path);
 	}
 }
diff --git a/src/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/docs-assembler/Navigation/GlobalNavigationFile.cs
@@ -109,8 +109,8 @@ private static IReadOnlyCollection<TableOfContentsReference> ReadChildren(YamlSt
 					if (source.AsSpan().IndexOf("://") == -1)
 					{
 						parent = source;
-						source = $"{NarrativeRepository.RepositoryName}://{source}";
 						pathPrefix = source;
+						source = $"{NarrativeRepository.RepositoryName}://{source}";
 					}
 
 					break;
diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml
@@ -43,11 +43,54 @@ toc:
       # repo is short for
       # - toc: <repo>://<parent_toc>
       #   path_prefix: <parent_toc>/<repo>
-      - repo: elasticsearch
-      - repo: apm
+      - repo: asciidocalypse
       - repo: apm-agent-android
-      - repo: apm-agent-go
       - repo: apm-agent-dotnet
+      - repo: apm-agent-go
+      - repo: apm-agent-ios
+      - repo: apm-agent-java
+      - repo: apm-agent-nodejs
+      - repo: apm-agent-php
+      - repo: apm-agent-python
+      - repo: apm-agent-ruby
+      - repo: apm-agent-rum-js
+      - repo: apm-aws-lambda
+      - repo: apm-k8s-attacher
+      - repo: beats
+      - repo: cloud-on-k8s
+      - repo: cloud
+      - repo: curator
+      - repo: ecctl
+      - repo: ecs-dotnet
+      - repo: ecs-logging-go-logrus
+      - repo: ecs-logging-go-zap
+      - repo: ecs-logging-go-zerolog
+      - repo: ecs-logging-java
+      - repo: ecs-logging-nodejs
+      - repo: ecs-logging-php
+      - repo: ecs-logging-python
+      - repo: ecs-logging-ruby
+      - repo: ecs-logging
+      - repo: ecs
+      - repo: eland
+      - repo: elastic-serverless-forwarder
+      - repo: elasticsearch-hadoop
+      - repo: elasticsearch-java
+      - repo: elasticsearch-js
+      - repo: elasticsearch-net
+      - repo: elasticsearch-php
+      - repo: elasticsearch-py
+      - repo: elasticsearch-rs
+      - repo: elasticsearch-ruby
+      - repo: elasticsearch
+      - repo: go-elasticsearch
+      - repo: integrations
+      - repo: kibana
+      - repo: logstash-docs
+      - repo: logstash
+      - repo: search-ui
+      - repo: integration-docs
+      - repo: security-docs
 
   #############
   # REFERENCE #
@@ -175,6 +218,9 @@ toc:
       - toc: docs-content://reference/ingestion-tools
         path_prefix: reference/ingestion-tools
         children:
+          # Elasticsearch (TODO discuss with Colleen)
+          - toc: elasticsearch://reference/ingestion-tools/enrich-processor
+            path_prefix: reference/elasticsearch/enrich-processor
 
           # Fleet and Elastic Agent
           # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/fleet/toc.yml

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ protected override void HandleItem(Diagnostic diagnostic)`
`24`	`24`	`_errors.Add(diagnostic);`
`25`	`25`	`else if (diagnostic.Severity == Severity.Warning)`
`26`	`26`	`_warnings.Add(diagnostic);`
`27`		`- else`
	`27`	`+ else if (!NoHints)`
`28`	`28`	`_hints.Add(diagnostic);`
`29`	`29`	`}`
`30`	`30`
Original file line number	Diff line number	Diff line change
`@@ -85,10 +85,12 @@ public class DiagnosticsCollector(IReadOnlyCollection<IDiagnosticsOutput> output`
`85`	`85`
`86`	`86`	`public HashSet<string> OffendingFiles { get; } = [];`
`87`	`87`
`88`		`- public HashSet<string> InUseSubstitutionKeys { get; } = [];`
	`88`	`+ public ConcurrentDictionary<string, bool> InUseSubstitutionKeys { get; } = [];`
`89`	`89`
`90`	`90`	`public ConcurrentBag<string> CrossLinks { get; } = [];`
`91`	`91`
	`92`	`+ public bool NoHints { get; init; }`
	`93`	`+`
`92`	`94`	`public Task StartAsync(Cancel cancellationToken)`
`93`	`95`	`{`
`94`	`96`	`if (_started is not null)`
`@@ -117,6 +119,8 @@ void Drain()`
`117`	`119`	`{`
`118`	`120`	`while (Channel.Reader.TryRead(out var item))`
`119`	`121`	`{`
	`122`	`+ if (item.Severity == Severity.Hint && NoHints)`
	`123`	`+ continue;`
`120`	`124`	`IncrementSeverityCount(item);`
`121`	`125`	`HandleItem(item);`
`122`	`126`	`_ = OffendingFiles.Add(item.File);`
`@@ -132,7 +136,7 @@ private void IncrementSeverityCount(Diagnostic item)`
`132`	`136`	`_ = Interlocked.Increment(ref _errors);`
`133`	`137`	`else if (item.Severity == Severity.Warning)`
`134`	`138`	`_ = Interlocked.Increment(ref _warnings);`
`135`		`- else if (item.Severity == Severity.Hint)`
	`139`	`+ else if (item.Severity == Severity.Hint && !NoHints)`
`136`	`140`	`_ = Interlocked.Increment(ref _hints);`
`137`	`141`	`}`
`138`	`142`
`@@ -175,5 +179,5 @@ public async ValueTask DisposeAsync()`
`175`	`179`	`}`
`176`	`180`
`177`	`181`	`public void CollectUsedSubstitutionKey(ReadOnlySpan<char> key) =>`
`178`		`- _ = InUseSubstitutionKeys.Add(key.ToString());`
	`182`	`+ _ = InUseSubstitutionKeys.TryAdd(key.ToString(), true);`
`179`	`183`	`}`
Original file line number	Diff line number	Diff line change
`@@ -91,7 +91,7 @@ public DocumentationSet(BuildContext build, ILoggerFactory logger, ICrossLinkRes`
`91`	`91`	`.SelectMany(extension => extension.ScanDocumentationFiles(ScanDocumentationFiles, DefaultFileHandling))`
`92`	`92`	`.ToArray();`
`93`	`93`
`94`		`- Files = files.Concat(additionalSources).ToArray();`
	`94`	`+ Files = files.Concat(additionalSources).Where(f => f is not ExcludedFile).ToArray();`
`95`	`95`
`96`	`96`	`LastWrite = Files.Max(f => f.SourceFile.LastWriteTimeUtc);`
`97`	`97`
`@@ -154,7 +154,7 @@ private DocumentationFile DefaultFileHandling(IFileInfo file, IDirectoryInfo sou`
`154`	`154`	`if (documentationFile is not null)`
`155`	`155`	`return documentationFile;`
`156`	`156`	`}`
`157`		`- return new StaticFile(file, sourceDirectory);`
	`157`	`+ return new ExcludedFile(file, sourceDirectory);`
`158`	`158`	`}`
`159`	`159`
`160`	`160`	`private void ValidateRedirectsExists()`
Original file line number	Diff line number	Diff line change
`@@ -109,8 +109,8 @@ private static IReadOnlyCollection<TableOfContentsReference> ReadChildren(YamlSt`
`109`	`109`	`if (source.AsSpan().IndexOf("://") == -1)`
`110`	`110`	`{`
`111`	`111`	`parent = source;`
`112`		`- source = $"{NarrativeRepository.RepositoryName}://{source}";`
`113`	`112`	`pathPrefix = source;`
	`113`	`+ source = $"{NarrativeRepository.RepositoryName}://{source}";`
`114`	`114`	`}`
`115`	`115`
`116`	`116`	`break;`