Cross-links: differentiate URL format and htmx behavior per build type

src/Elastic.Codex/Building/CodexBuildService.cs

@@ -11,6 +11,7 @@
 using Elastic.Documentation.Configuration.Codex;
 using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Isolated;
+using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links.CrossLinks;
 using Elastic.Documentation.Navigation;
 using Elastic.Documentation.Navigation.Isolated.Node;
@@ -60,10 +61,13 @@ public async Task<CodexBuildResult> BuildAll(
 		var documentationSets = new Dictionary<string, IDocumentationSetNavigation>();
 		var buildContexts = new List<CodexDocumentationSetBuildContext>();
 
+		var environment = context.Configuration.Environment ?? "internal";
+		using var codexLinkIndexReader = new GitLinkIndexReader(environment, context.ReadFileSystem);
+
 		// Phase 1: Load and parse all documentation sets
 		foreach (var checkout in cloneResult.Checkouts)
 		{
-			var buildContext = await LoadDocumentationSet(context, checkout, fileSystem, ctx);
+			var buildContext = await LoadDocumentationSet(context, checkout, fileSystem, codexLinkIndexReader, ctx);
 			if (buildContext != null)
 			{
 				buildContexts.Add(buildContext);
@@ -119,6 +123,7 @@ public async Task<CodexBuildResult> BuildAll(
 		CodexContext context,
 		CodexCheckout checkout,
 		IFileSystem fileSystem,
+		ILinkIndexReader codexLinkIndexReader,
 		Cancel ctx)
 	{
 		_logger.LogInformation("Loading documentation set: {Name}", checkout.Reference.Name);
@@ -170,8 +175,23 @@ public async Task<CodexBuildResult> BuildAll(
 				BuildType = BuildType.Codex
 			};
 
-			// Create cross-link resolver (simplified for codex - no external links)
-			var crossLinkResolver = NoopCrossLinkResolver.Instance;
+			ICrossLinkResolver crossLinkResolver;
+			if (buildContext.Configuration.CrossLinkEntries.Length > 0)
+			{
+				var fetcher = new DocSetConfigurationCrossLinkFetcher(
+					logFactory,
+					buildContext.Configuration,
+					codexLinkIndexReader: buildContext.Configuration.Registry != DocSetRegistry.Public ? codexLinkIndexReader : null);
+				var crossLinks = await fetcher.FetchCrossLinks(ctx);
+				IUriEnvironmentResolver? uriResolver = crossLinks.CodexRepositories is not null
+					? new CodexAwareUriResolver(crossLinks.CodexRepositories, useRelativePaths: true)
+					: null;
+				crossLinkResolver = new CrossLinkResolver(crossLinks, uriResolver);
+			}
+			else
+			{
+				crossLinkResolver = NoopCrossLinkResolver.Instance;
+			}
 
 			// Create documentation set
 			var documentationSet = new DocumentationSet(buildContext, logFactory, crossLinkResolver);

src/Elastic.Documentation.Configuration/Builder/ConfigurationFile.cs

@@ -27,6 +27,16 @@ public record ConfigurationFile
 
 	public string[] CrossLinkRepositories { get; } = [];
 
+	/// <summary>
+	/// Registry for this documentation set. <c>Public</c> uses S3 link index; other values use codex-link-index.
+	/// </summary>
+	public DocSetRegistry Registry { get; } = DocSetRegistry.Public;
+
+	/// <summary>
+	/// Parsed cross-link entries with registry for each target.
+	/// </summary>
+	public CrossLinkEntry[] CrossLinkEntries { get; } = [];
+
 	/// The maximum depth `toc.yml` files may appear
 	public int MaxTocDepth { get; } = 1;
 
@@ -90,8 +100,23 @@ public ConfigurationFile(DocumentationSetFile docSetFile, IDocumentationSetConte
 			Exclude = [.. docSetFile.Exclude.Where(s => !string.IsNullOrEmpty(s) && !s.StartsWith('!')).Select(Glob.Parse)];
 			Include = [.. docSetFile.Exclude.Where(s => !string.IsNullOrEmpty(s) && s.StartsWith('!')).Select(s => s.TrimStart('!'))];
 
-			// Set cross link repositories
-			CrossLinkRepositories = [.. docSetFile.CrossLinks];
+			// Parse registry (null/empty/"public" -> Public)
+			var registry = DocSetRegistry.Public;
+			if (!string.IsNullOrWhiteSpace(docSetFile.Registry) &&
+				DocSetRegistryExtensions.TryParse(docSetFile.Registry.Trim(), out var parsedRegistry, true))
+				registry = parsedRegistry;
+
+			Registry = registry;
+
+			// Parse cross-link entries with optional registry prefix (e.g. public://elasticsearch)
+			CrossLinkEntries = docSetFile.CrossLinks
+				.Where(raw => !string.IsNullOrWhiteSpace(raw))
+				.Select(raw => ParseCrossLinkEntry(raw.Trim(), registry, context.ConfigurationPath, context))
+				.Where(entry => entry is not null)
+				.Select(entry => entry!)
+				.ToArray();
+
+			CrossLinkRepositories = CrossLinkEntries.Select(e => e.Repository).ToArray();
 
 			// Extensions - assuming they're not in DocumentationSetFile yet
 			Extensions = new EnabledExtensions(docSetFile.Extensions);
@@ -159,4 +184,39 @@ public ConfigurationFile(DocumentationSetFile docSetFile, IDocumentationSetConte
 		}
 	}
 
+	private static CrossLinkEntry? ParseCrossLinkEntry(string raw, DocSetRegistry docsetRegistry, IFileInfo configPath, IDocumentationContext context)
+	{
+		DocSetRegistry entryRegistry;
+		string repository;
+
+		var colonSlash = raw.IndexOf("://", StringComparison.Ordinal);
+		if (colonSlash >= 0)
+		{
+			var prefix = raw[..colonSlash];
+			repository = raw[(colonSlash + 3)..];
+			if (string.IsNullOrWhiteSpace(repository))
+			{
+				context.EmitError(configPath, $"Cross-link '{raw}' has empty repository after registry prefix.");
+				return null;
+			}
+			if (!DocSetRegistryExtensions.TryParse(prefix, out entryRegistry, true))
+			{
+				context.EmitError(configPath, $"Cross-link '{raw}' uses unknown registry '{prefix}'. Use 'public' or 'internal'.");
+				return null;
+			}
+		}
+		else
+		{
+			repository = raw;
+			entryRegistry = docsetRegistry;
+		}
+
+		if (docsetRegistry == DocSetRegistry.Public && entryRegistry != DocSetRegistry.Public)
+		{
+			context.EmitError(configPath, $"Public documentation cannot link to codex docs. Cross-link '{raw}' targets registry '{entryRegistry.ToStringFast()}'. Remove it or use a public docset.");
+			return null;
+		}
+
+		return new CrossLinkEntry(repository, entryRegistry);
+	}
 }

src/Elastic.Documentation.Configuration/CrossLinkEntry.cs

@@ -0,0 +1,12 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Documentation.Configuration;
+
+/// <summary>
+/// A parsed cross-link entry from docset.yml, with the target registry for lookup.
+/// </summary>
+/// <param name="Repository">Repository name (e.g. elasticsearch, docs-eng-team).</param>
+/// <param name="Registry">Registry to use for lookup (public S3 or codex environment).</param>
+public record CrossLinkEntry(string Repository, DocSetRegistry Registry);

src/Elastic.Documentation.Configuration/DocSetRegistry.cs

@@ -0,0 +1,24 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.ComponentModel.DataAnnotations;
+using NetEscapades.EnumGenerators;
+
+namespace Elastic.Documentation.Configuration;
+
+/// <summary>
+/// Registry type for cross-link resolution. Maps to the link index source:
+/// <c>Public</c> uses the S3-based public link index; other values use the codex-link-index for that environment.
+/// </summary>
+[EnumExtensions]
+public enum DocSetRegistry
+{
+	/// <summary>Public documentation; uses S3-based link index.</summary>
+	[Display(Name = "public")]
+	Public,
+
+	/// <summary>Internal codex environment; uses codex-link-index/internal/.</summary>
+	[Display(Name = "internal")]
+	Internal,
+}

src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj

@@ -14,6 +14,7 @@
 
   <ItemGroup>
     <PackageReference Include="DotNet.Glob" />
+    <PackageReference Include="NetEscapades.EnumGenerators" />
     <PackageReference Include="Samboy063.Tomlet" />
     <PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator"/>
     <PackageReference Include="YamlDotNet"/>

src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs

@@ -20,11 +20,25 @@ public record FetchedCrossLinks
 
 	public required FrozenDictionary<string, LinkRegistryEntry> LinkIndexEntries { get; init; }
 
+	/// <summary>
+	/// Optional map of repository name to link index registry URL for error messages.
+	/// When null or missing, falls back to the public S3 URL.
+	/// </summary>
+	public FrozenDictionary<string, string>? RegistryUrlsByRepository { get; init; }
+
+	/// <summary>
+	/// Set of repository names that belong to a codex (non-public) registry.
+	/// Used by the URI resolver to generate codex URLs instead of public preview URLs.
+	/// </summary>
+	public FrozenSet<string>? CodexRepositories { get; init; }
+
 	public static FetchedCrossLinks Empty { get; } = new()
 	{
 		DeclaredRepositories = [],
 		LinkReferences = new Dictionary<string, RepositoryLinks>().ToFrozenDictionary(),
-		LinkIndexEntries = new Dictionary<string, LinkRegistryEntry>().ToFrozenDictionary()
+		LinkIndexEntries = new Dictionary<string, LinkRegistryEntry>().ToFrozenDictionary(),
+		RegistryUrlsByRepository = null,
+		CodexRepositories = null
 	};
 }
 
@@ -87,21 +101,47 @@ protected async Task<RepositoryLinks> FetchCrossLinks(string repository, string[
 		throw new Exception($"Repository found in link index however none of: '{string.Join(", ", keys)}' branches found");
 	}
 
-	protected async Task<RepositoryLinks> FetchLinkIndexEntry(string repository, LinkRegistryEntry linkRegistryEntry, Cancel ctx)
+	protected Task<RepositoryLinks> FetchLinkIndexEntry(string repository, LinkRegistryEntry linkRegistryEntry, Cancel ctx) =>
+		FetchLinkIndexEntryFromReader(linkIndexProvider, repository, linkRegistryEntry, ctx);
+
+	/// <summary>
+	/// Fetches repository links from a specific reader. Used for dual-registry (public + codex) fetching.
+	/// </summary>
+	protected async Task<RepositoryLinks> FetchLinkIndexEntryFromReader(
+		ILinkIndexReader reader,
+		string repository,
+		LinkRegistryEntry linkRegistryEntry,
+		Cancel ctx)
 	{
 		var linkReference = await TryGetCachedLinkReference(repository, linkRegistryEntry);
 		if (linkReference is not null)
 		{
-			Logger.LogInformation("Using locally cached links.json for '{Repository}' from {RegistryUrl}", repository, linkIndexProvider.RegistryUrl);
+			Logger.LogInformation("Using locally cached links.json for '{Repository}' from {RegistryUrl}", repository, reader.RegistryUrl);
 			return linkReference;
 		}
 
-		Logger.LogInformation("Fetching links.json for '{Repository}' from {RegistryUrl}", repository, linkIndexProvider.RegistryUrl);
-		linkReference = await linkIndexProvider.GetRepositoryLinks(linkRegistryEntry.Path, ctx);
+		Logger.LogInformation("Fetching links.json for '{Repository}' from {RegistryUrl}", repository, reader.RegistryUrl);
+		linkReference = await reader.GetRepositoryLinks(linkRegistryEntry.Path, ctx);
 		WriteLinksJsonCachedFile(repository, linkRegistryEntry, linkReference);
 		return linkReference;
 	}
 
+	/// <summary>
+	/// Fetches cross-links for a repository from a specific reader. Used for dual-registry fetching.
+	/// </summary>
+	protected static async Task<RepositoryLinks> FetchCrossLinksFromReader(
+		ILinkIndexReader reader,
+		string repository,
+		CrossLinkFetcher fetcher,
+		Cancel ctx)
+	{
+		var linkIndex = await reader.GetRegistry(ctx);
+		if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks))
+			throw new Exception($"Repository {repository} not found in link index");
+		var entry = GetNextContentSourceLinkIndexEntry(repositoryLinks, repository);
+		return await fetcher.FetchLinkIndexEntryFromReader(reader, repository, entry, ctx);
+	}
+
 	private void WriteLinksJsonCachedFile(string repository, LinkRegistryEntry linkRegistryEntry, RepositoryLinks linkReference)
 	{
 		var cachedFileName = $"links-elastic-{repository}-{linkRegistryEntry.Branch}-{linkRegistryEntry.ETag}.json";

src/Elastic.Documentation.Links/CrossLinks/CrossLinkResolver.cs

@@ -75,7 +75,7 @@ public static bool TryResolve(
 				return true;
 			}
 
-			errorEmitter($"'{crossLinkUri.Scheme}' was not found in the cross link index");
+			errorEmitter($"'{crossLinkUri.Scheme}' was not found in the cross link index. Ensure it is listed under 'cross_links' in your docset.yml");
 			return false;
 		}
 
@@ -89,9 +89,12 @@ public static bool TryResolve(
 		if (sourceLinkReference.Links.TryGetValue(originalLookupPath, out var directLinkMetadata))
 			return ResolveDirectLink(errorEmitter, uriResolver, crossLinkUri, originalLookupPath, directLinkMetadata, out resolvedUri);
 
-		var linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{crossLinkUri.Scheme}/main/links.json";
-		if (fetchedCrossLinks.LinkIndexEntries.TryGetValue(crossLinkUri.Scheme, out var indexEntry))
-			linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{indexEntry.Path}";
+		var registryUrl = fetchedCrossLinks.RegistryUrlsByRepository?.GetValueOrDefault(crossLinkUri.Scheme)
+			?? "https://elastic-docs-link-index.s3.us-east-2.amazonaws.com";
+		var baseUrl = GetLinksJsonBaseUrl(registryUrl);
+		var linksJson = fetchedCrossLinks.LinkIndexEntries.TryGetValue(crossLinkUri.Scheme, out var indexEntry)
+			? $"{baseUrl}/{indexEntry.Path}"
+			: $"{baseUrl}/elastic/{crossLinkUri.Scheme}/main/links.json";
 
 		errorEmitter($"'{originalLookupPath}' is not a valid link in the '{crossLinkUri.Scheme}' cross link index: {linksJson}");
 		resolvedUri = null;
@@ -231,4 +234,14 @@ public static string ToTargetUrlPath(string lookupPath)
 			path = string.Empty;
 		return path;
 	}
+
+	/// <summary>Derives the base URL for links.json from a reader's RegistryUrl (S3 or GitHub).</summary>
+	private static string GetLinksJsonBaseUrl(string registryUrl)
+	{
+		if (registryUrl.Contains("github.com", StringComparison.OrdinalIgnoreCase))
+			return $"{registryUrl.TrimEnd('/')}/blob/main";
+		if (registryUrl.Contains("/link-index.json", StringComparison.OrdinalIgnoreCase))
+			return registryUrl.Replace("/link-index.json", "", StringComparison.OrdinalIgnoreCase).TrimEnd('/');
+		return registryUrl.TrimEnd('/');
+	}
 }

src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs

@@ -3,51 +3,72 @@
 // See the LICENSE file in the project root for more information
 
 using System.Collections.Frozen;
+using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.Builder;
 using Elastic.Documentation.LinkIndex;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Documentation.Links.CrossLinks;
 
 /// Fetches cross-links from all the declared repositories in the docset.yml configuration see <see cref="ConfigurationFile"/>
-public class DocSetConfigurationCrossLinkFetcher(ILoggerFactory logFactory, ConfigurationFile configuration, ILinkIndexReader? linkIndexProvider = null)
+public class DocSetConfigurationCrossLinkFetcher(
+	ILoggerFactory logFactory,
+	ConfigurationFile configuration,
+	ILinkIndexReader? linkIndexProvider = null,
+	ILinkIndexReader? codexLinkIndexReader = null)
 	: CrossLinkFetcher(logFactory, linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous())
 {
 	private readonly ILogger _logger = logFactory.CreateLogger(nameof(DocSetConfigurationCrossLinkFetcher));
+	private readonly ILinkIndexReader? _codexReader = codexLinkIndexReader;
 
 	public override async Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx)
 	{
 		Logger.LogInformation("Fetching cross-links for all repositories defined in docset.yml");
 		var linkReferences = new Dictionary<string, RepositoryLinks>();
 		var linkIndexEntries = new Dictionary<string, LinkRegistryEntry>();
+		var registryUrlsByRepository = new Dictionary<string, string>();
+		var codexRepositories = new HashSet<string>();
 		var declaredRepositories = new HashSet<string>();
 
-		foreach (var repository in configuration.CrossLinkRepositories)
+		var publicReader = linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous();
+		var useDualRegistry = configuration.Registry != DocSetRegistry.Public && _codexReader is not null;
+
+		foreach (var entry in configuration.CrossLinkEntries)
 		{
-			_ = declaredRepositories.Add(repository);
+			_ = declaredRepositories.Add(entry.Repository);
+			var isCodexEntry = useDualRegistry && entry.Registry != DocSetRegistry.Public;
+			var reader = isCodexEntry ? _codexReader! : publicReader;
+
+			if (isCodexEntry)
+				_ = codexRepositories.Add(entry.Repository);
+
 			try
 			{
-				var linkReference = await FetchCrossLinks(repository, ["main", "master"], ctx);
-				linkReferences.Add(repository, linkReference);
+				var linkReference = await FetchCrossLinksFromReader(reader, entry.Repository, this, ctx);
+				linkReferences.Add(entry.Repository, linkReference);
+				registryUrlsByRepository[entry.Repository] = reader.RegistryUrl;
 
-				var linkIndexReference = await GetLinkIndexEntry(repository, ctx);
-				linkIndexEntries.Add(repository, linkIndexReference);
+				var registry = await reader.GetRegistry(ctx);
+				if (registry.Repositories.TryGetValue(entry.Repository, out var repoBranches))
+				{
+					var linkIndexEntry = GetNextContentSourceLinkIndexEntry(repoBranches, entry.Repository);
+					linkIndexEntries.Add(entry.Repository, linkIndexEntry);
+				}
 			}
 			catch (Exception ex)
 			{
-				// Log the error but continue processing other repositories
-				_logger.LogWarning(ex, "Error fetching link data for repository '{Repository}'. Cross-links to this repository may not resolve correctly.", repository);
+				_logger.LogWarning(ex, "Error fetching link data for repository '{Repository}'. Cross-links to this repository may not resolve correctly.", entry.Repository);
+				_ = registryUrlsByRepository.TryAdd(entry.Repository, reader.RegistryUrl);
 
-				// Add an empty entry so we at least recognize the repository exists
-				if (!linkReferences.ContainsKey(repository))
+				if (!linkReferences.ContainsKey(entry.Repository))
 				{
-					linkReferences.Add(repository, new RepositoryLinks
+					linkReferences.Add(entry.Repository, new RepositoryLinks
 					{
 						Links = [],
 						Origin = new GitCheckoutInformation
 						{
 							Branch = "main",
-							RepositoryName = repository,
+							RepositoryName = entry.Repository,
 							Remote = "origin",
 							Ref = "refs/heads/main"
 						},
@@ -63,6 +84,8 @@ public override async Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx)
 			DeclaredRepositories = declaredRepositories,
 			LinkReferences = linkReferences.ToFrozenDictionary(),
 			LinkIndexEntries = linkIndexEntries.ToFrozenDictionary(),
+			RegistryUrlsByRepository = registryUrlsByRepository.ToFrozenDictionary(),
+			CodexRepositories = codexRepositories.Count > 0 ? codexRepositories.ToFrozenSet() : null,
 		};
 	}
 }