From 199648d7ed53ec65f2f86421f3572aa6805697dc Mon Sep 17 00:00:00 2001 From: ptlthg <24925519+ptlthg@users.noreply.github.com> Date: Wed, 11 Feb 2026 18:38:25 -0500 Subject: [PATCH 1/2] Remove unused caches on db leaderboards --- .../Leaderboards/Services/LbService.cs | 162 ++++-------------- .../Services/LeaderboardCacheMetrics.cs | 61 +------ 2 files changed, 39 insertions(+), 184 deletions(-) diff --git a/EliteAPI/Features/Leaderboards/Services/LbService.cs b/EliteAPI/Features/Leaderboards/Services/LbService.cs index bc7c60d2..a0f19a3e 100644 --- a/EliteAPI/Features/Leaderboards/Services/LbService.cs +++ b/EliteAPI/Features/Leaderboards/Services/LbService.cs @@ -10,7 +10,6 @@ using EliteAPI.Utilities; using FastEndpoints; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Caching.Hybrid; using StackExchange.Redis; namespace EliteAPI.Features.Leaderboards.Services; @@ -69,71 +68,10 @@ public class LbService( ILeaderboardRegistrationService registrationService, ILogger logger, IConnectionMultiplexer redis, - HybridCache cache, - ILeaderboardCacheMetrics cacheMetrics, IMemberService memberService, DataContext context) : ILbService { - /// - /// Cached anchor data for leaderboard rank lookups - /// - private record CachedAnchor(decimal Score, int EntryId); - - /// - /// Cached anchor data for score-based lookups (stores rank at a score threshold) - /// - private record CachedAmountAnchor(int Rank); - - /// - /// Get the bucket size for a given rank. Higher ranks get finer granularity. - /// Buckets must be larger than typical upcoming request sizes (100) to have any chance of cache hits. - /// - private static int GetRankBucket(int atRank) { - return atRank switch { - <= 1000 => 10, // Top 1k: bucket of 10, 100 possible buckets - <= 5000 => 50, // Top 5k: bucket of 50, 80 possible buckets - <= 25000 => 250, // Top 25k: bucket of 250, 80 possible buckets - <= 50000 => 500, // Top 50k: bucket of 500, 50 possible buckets - <= 100000 => 1000, // Top 100k: bucket of 1000, 50 possible buckets - _ => 2500 // 100k+: bucket of 2500 - }; - } - - /// - /// Round a rank to its bucket boundary - /// - private static int GetBucketedRank(int atRank) { - var bucket = GetRankBucket(atRank); - if (bucket == 1) return atRank; - return (int)Math.Ceiling((double)atRank / bucket) * bucket; - } - - /// - /// Get the cache TTL for a given rank. Higher ranks get shorter TTLs for fresher data. - /// - private static HybridCacheEntryOptions GetCacheOptions(int atRank) { - var ttl = atRank switch { - <= 1000 => TimeSpan.FromSeconds(20), - <= 5000 => TimeSpan.FromSeconds(30), - <= 25000 => TimeSpan.FromSeconds(45), - <= 50000 => TimeSpan.FromSeconds(60), - _ => TimeSpan.FromSeconds(120) - }; - return new HybridCacheEntryOptions { - Expiration = ttl, - LocalCacheExpiration = TimeSpan.FromSeconds(Math.Min(ttl.TotalSeconds, 15)) - }; - } - - /// - /// Round upcoming count to standard bucket sizes for better cache hit rates. - /// - private static int GetBucketedUpcoming(int upcoming) { - // Maximize cache reuse, a switch statmement might be used later - return 100; - } - public async Task<(Leaderboard? lb, ILeaderboardDefinition? definition)> GetLeaderboard(string leaderboardId) { if (!registrationService.LeaderboardsById.TryGetValue(leaderboardId, out var definition)) return (null, null); @@ -1205,34 +1143,18 @@ await memberService.UpdatePlayerIfNeeded(member.PlayerUuid, RequestedResources.P var usingAtAmount = atAmount is > 0 && (decimal)atAmount > userScore; if (atRank is > 0 && (anchorRank == -1 || atRank < anchorRank)) { - var bucketedRank = GetBucketedRank(atRank.Value); - var anchorCacheKey = - $"lb:anchor:{leaderboardId}:{gameMode ?? "all"}:{identifier ?? "c"}:{removedFilter}:{bucketedRank}"; - var cacheOptions = GetCacheOptions(bucketedRank); - - var anchorCacheMiss = false; - var cachedAnchor = await cache.GetOrCreateAsync(anchorCacheKey, async ct => { - anchorCacheMiss = true; - var anchor = await baseQuery - .OrderByDescending(e => e.Score) - .ThenByDescending(e => e.LeaderboardEntryId) - .Skip(bucketedRank - 1) - .Select(e => new CachedAnchor(e.Score, e.LeaderboardEntryId)) - .FirstOrDefaultAsync(ct); - return anchor; - }, cacheOptions, cancellationToken: cancellationToken); - - if (anchorCacheMiss) { - cacheMetrics.RecordAnchorCacheMiss(leaderboardId, bucketedRank); - } - else { - cacheMetrics.RecordAnchorCacheHit(leaderboardId, bucketedRank); - } - - if (cachedAnchor is not null) { - anchorScore = cachedAnchor.Score; - anchorId = cachedAnchor.EntryId; - anchorRank = bucketedRank; + var targetRank = atRank.Value; + var anchorEntry = await baseQuery + .OrderByDescending(e => e.Score) + .ThenByDescending(e => e.LeaderboardEntryId) + .Skip(targetRank - 1) + .Select(e => new { e.Score, e.LeaderboardEntryId }) + .FirstOrDefaultAsync(cancellationToken); + + if (anchorEntry is not null) { + anchorScore = anchorEntry.Score; + anchorId = anchorEntry.LeaderboardEntryId; + anchorRank = targetRank; } } else if (usingAtAmount && atAmount is not null) { @@ -1246,41 +1168,33 @@ await memberService.UpdatePlayerIfNeeded(member.PlayerUuid, RequestedResources.P var previousPlayers = new List(); if (upcoming > 0 && !userEntry.IsRemoved) { - var bucketedUpcoming = GetBucketedUpcoming(upcoming.Value); - var upcomingCacheKey = - $"lb:upcoming:{leaderboardId}:{gameMode ?? "all"}:{identifier ?? "c"}:{removedFilter}:{definition.IsMemberLeaderboard()}:{anchorRank}:{bucketedUpcoming}"; - var cacheOptions = GetCacheOptions(anchorRank > 0 ? anchorRank : 50000); - - var upcomingCacheMiss = false; - var cachedUpcoming = await cache.GetOrCreateAsync(upcomingCacheKey, async ct => { - upcomingCacheMiss = true; - var upcomingQuery = baseQuery - .Where(e => e.Score > anchorScore || (e.Score == anchorScore && e.LeaderboardEntryId > anchorId)) - .OrderBy(e => e.Score) - .ThenBy(e => e.LeaderboardEntryId) - .Take(bucketedUpcoming); - - var list = definition.IsMemberLeaderboard() - ? await upcomingQuery.MapToMemberLeaderboardEntries(includeMeta: false).ToListAsync(ct) - : await upcomingQuery.MapToProfileLeaderboardEntries(removedFilter).ToListAsync(ct); - - return list.Select((entry, i) => entry.ToDtoWithRank(anchorRank - i - 1)).ToList(); - }, cacheOptions, cancellationToken: cancellationToken); - - if (upcomingCacheMiss) { - cacheMetrics.RecordUpcomingCacheMiss(leaderboardId, anchorRank > 0 ? anchorRank : 50000); - } - else { - cacheMetrics.RecordUpcomingCacheHit(leaderboardId, anchorRank > 0 ? anchorRank : 50000); + var fetchLimit = usingAtAmount ? upcoming.Value : upcoming.Value + 1; + var upcomingQuery = baseQuery + .Where(e => e.Score > anchorScore || (e.Score == anchorScore && e.LeaderboardEntryId > anchorId)); + if (usingAtAmount && atAmount is not null) { + var atAmountDecimal = (decimal)atAmount.Value; + upcomingQuery = upcomingQuery.Where(e => e.Score > atAmountDecimal); } - if (usingAtAmount) { - upcomingPlayers = cachedUpcoming.Where(e => atAmount < e.Amount && e.Uuid != resourceId) - .Take(upcoming.Value).ToList(); - } - else { - upcomingPlayers = cachedUpcoming.Where(e => e.Uuid != resourceId).Take(upcoming.Value).ToList(); - } + var upcomingList = definition.IsMemberLeaderboard() + ? await upcomingQuery + .OrderBy(e => e.Score) + .ThenBy(e => e.LeaderboardEntryId) + .Take(fetchLimit) + .MapToMemberLeaderboardEntries(includeMeta: false) + .ToListAsync(cancellationToken) + : await upcomingQuery + .OrderBy(e => e.Score) + .ThenBy(e => e.LeaderboardEntryId) + .Take(fetchLimit) + .MapToProfileLeaderboardEntries(removedFilter) + .ToListAsync(cancellationToken); + + upcomingPlayers = upcomingList + .Select((entry, i) => entry.ToDtoWithRank(anchorRank - i - 1)) + .Where(e => e.Uuid != resourceId) + .Take(upcoming.Value) + .ToList(); var next = upcomingPlayers.FirstOrDefault(); if (next is not null && next.Rank > 0) { @@ -1597,4 +1511,4 @@ public class LeaderboardEntry public class LeaderboardEntryWithRank : LeaderboardEntry { public int Rank { get; init; } -} \ No newline at end of file +} diff --git a/EliteAPI/Features/Leaderboards/Services/LeaderboardCacheMetrics.cs b/EliteAPI/Features/Leaderboards/Services/LeaderboardCacheMetrics.cs index 905c729b..31efc14d 100644 --- a/EliteAPI/Features/Leaderboards/Services/LeaderboardCacheMetrics.cs +++ b/EliteAPI/Features/Leaderboards/Services/LeaderboardCacheMetrics.cs @@ -4,10 +4,6 @@ namespace EliteAPI.Features.Leaderboards.Services; public interface ILeaderboardCacheMetrics { - void RecordAnchorCacheHit(string leaderboardId, int rankTier); - void RecordAnchorCacheMiss(string leaderboardId, int rankTier); - void RecordUpcomingCacheHit(string leaderboardId, int rankTier); - void RecordUpcomingCacheMiss(string leaderboardId, int rankTier); void RecordBatchProcessed(int count, double durationMs); void RecordQueueDepth(int depth); void RecordUpdateDropped(); @@ -15,10 +11,6 @@ public interface ILeaderboardCacheMetrics public class LeaderboardCacheMetrics : ILeaderboardCacheMetrics { - private readonly Counter _anchorCacheHits; - private readonly Counter _anchorCacheMisses; - private readonly Counter _upcomingCacheHits; - private readonly Counter _upcomingCacheMisses; private readonly Counter _batchesProcessed; private readonly Histogram _batchDuration; private readonly Counter _entriesProcessed; @@ -27,23 +19,7 @@ public class LeaderboardCacheMetrics : ILeaderboardCacheMetrics public LeaderboardCacheMetrics(IMeterFactory meterFactory) { var meter = meterFactory.Create("eliteapi.leaderboard"); - - _anchorCacheHits = meter.CreateCounter( - "eliteapi.leaderboard.anchor_cache_hits", - description: "Number of anchor cache hits for leaderboard rank lookups"); - - _anchorCacheMisses = meter.CreateCounter( - "eliteapi.leaderboard.anchor_cache_misses", - description: "Number of anchor cache misses for leaderboard rank lookups"); - - _upcomingCacheHits = meter.CreateCounter( - "eliteapi.leaderboard.upcoming_cache_hits", - description: "Number of upcoming players cache hits"); - - _upcomingCacheMisses = meter.CreateCounter( - "eliteapi.leaderboard.upcoming_cache_misses", - description: "Number of upcoming players cache misses"); - + _batchesProcessed = meter.CreateCounter( "eliteapi.leaderboard.batches_processed", description: "Number of leaderboard update batches processed"); @@ -67,30 +43,6 @@ public LeaderboardCacheMetrics(IMeterFactory meterFactory) { description: "Current number of pending leaderboard updates in the queue"); } - public void RecordAnchorCacheHit(string leaderboardId, int rankTier) { - _anchorCacheHits.Add(1, - new KeyValuePair("leaderboard", leaderboardId), - new KeyValuePair("rank_tier", GetRankTierLabel(rankTier))); - } - - public void RecordAnchorCacheMiss(string leaderboardId, int rankTier) { - _anchorCacheMisses.Add(1, - new KeyValuePair("leaderboard", leaderboardId), - new KeyValuePair("rank_tier", GetRankTierLabel(rankTier))); - } - - public void RecordUpcomingCacheHit(string leaderboardId, int rankTier) { - _upcomingCacheHits.Add(1, - new KeyValuePair("leaderboard", leaderboardId), - new KeyValuePair("rank_tier", GetRankTierLabel(rankTier))); - } - - public void RecordUpcomingCacheMiss(string leaderboardId, int rankTier) { - _upcomingCacheMisses.Add(1, - new KeyValuePair("leaderboard", leaderboardId), - new KeyValuePair("rank_tier", GetRankTierLabel(rankTier))); - } - public void RecordBatchProcessed(int count, double durationMs) { _batchesProcessed.Add(1); _batchDuration.Record(durationMs); @@ -104,15 +56,4 @@ public void RecordQueueDepth(int depth) { public void RecordUpdateDropped() { _updatesDropped.Add(1); } - - private static string GetRankTierLabel(int rank) { - return rank switch { - <= 1000 => "top_1k", - <= 5000 => "top_5k", - <= 25000 => "top_25k", - <= 50000 => "top_50k", - <= 100000 => "top_100k", - _ => "100k_plus" - }; - } } From aabab7b4776b5c664d4cd7203bbfd79ed0c8ae94 Mon Sep 17 00:00:00 2001 From: ptlthg <24925519+ptlthg@users.noreply.github.com> Date: Wed, 11 Feb 2026 19:24:18 -0500 Subject: [PATCH 2/2] Add metrics for redis leaderboard memory usage --- .../LeaderboardRedisMemoryMetricsService.cs | 186 ++++++++++++++++++ .../Services/LeaderboardRedisSyncService.cs | 4 + EliteAPI/Utilities/ServiceExtensions.cs | 3 +- 3 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 EliteAPI/Features/Leaderboards/Services/LeaderboardRedisMemoryMetricsService.cs diff --git a/EliteAPI/Features/Leaderboards/Services/LeaderboardRedisMemoryMetricsService.cs b/EliteAPI/Features/Leaderboards/Services/LeaderboardRedisMemoryMetricsService.cs new file mode 100644 index 00000000..49828398 --- /dev/null +++ b/EliteAPI/Features/Leaderboards/Services/LeaderboardRedisMemoryMetricsService.cs @@ -0,0 +1,186 @@ +using System.Diagnostics.Metrics; +using StackExchange.Redis; + +namespace EliteAPI.Features.Leaderboards.Services; + +public class LeaderboardRedisMemoryMetricsService +{ + private const int MemoryUsageBatchSize = 256; + + private readonly IConnectionMultiplexer _redis; + private readonly ILogger _logger; + + private long _sortedSetBytes; + private long _memberHashBytes; + private long _minScoreBytes; + private long _totalBytes; + private long _sortedSetKeys; + private long _memberHashKeys; + private long _minScoreKeys; + private long _lastRefreshUnixSeconds; + private int _refreshSuccessful; + + public LeaderboardRedisMemoryMetricsService( + IConnectionMultiplexer redis, + IMeterFactory meterFactory, + ILogger logger) { + _redis = redis; + _logger = logger; + + var meter = meterFactory.Create("eliteapi.leaderboard"); + meter.CreateObservableGauge( + "eliteapi.leaderboard.redis_memory_bytes", + ObserveRedisMemoryBytes, + unit: "By", + description: "Redis memory used by leaderboard keys"); + meter.CreateObservableGauge( + "eliteapi.leaderboard.redis_keys", + ObserveRedisKeyCounts, + description: "Redis key counts used by leaderboard keys"); + meter.CreateObservableGauge( + "eliteapi.leaderboard.redis_memory_last_refresh_unix", + () => Interlocked.Read(ref _lastRefreshUnixSeconds), + unit: "s", + description: "Unix timestamp of the last successful leaderboard Redis memory sample"); + meter.CreateObservableGauge( + "eliteapi.leaderboard.redis_memory_refresh_success", + () => Interlocked.CompareExchange(ref _refreshSuccessful, 0, 0), + description: "1 when the last leaderboard Redis memory sample succeeded, 0 otherwise"); + } + + private IEnumerable> ObserveRedisMemoryBytes() { + yield return new Measurement( + Interlocked.Read(ref _totalBytes), + new KeyValuePair("component", "total")); + yield return new Measurement( + Interlocked.Read(ref _sortedSetBytes), + new KeyValuePair("component", "sorted_set")); + yield return new Measurement( + Interlocked.Read(ref _memberHashBytes), + new KeyValuePair("component", "member_hash")); + yield return new Measurement( + Interlocked.Read(ref _minScoreBytes), + new KeyValuePair("component", "min_score")); + } + + private IEnumerable> ObserveRedisKeyCounts() { + yield return new Measurement( + Interlocked.Read(ref _sortedSetKeys) + Interlocked.Read(ref _memberHashKeys) + + Interlocked.Read(ref _minScoreKeys), + new KeyValuePair("component", "total")); + yield return new Measurement( + Interlocked.Read(ref _sortedSetKeys), + new KeyValuePair("component", "sorted_set")); + yield return new Measurement( + Interlocked.Read(ref _memberHashKeys), + new KeyValuePair("component", "member_hash")); + yield return new Measurement( + Interlocked.Read(ref _minScoreKeys), + new KeyValuePair("component", "min_score")); + } + + public async Task RefreshAsync(CancellationToken ct) { + try { + var endPoint = _redis.GetEndPoints().FirstOrDefault(); + if (endPoint is null) { + Interlocked.Exchange(ref _refreshSuccessful, 0); + return; + } + + var server = _redis.GetServer(endPoint); + if (!server.IsConnected) { + Interlocked.Exchange(ref _refreshSuccessful, 0); + return; + } + + var db = _redis.GetDatabase(); + var dbIndex = db.Database >= 0 ? db.Database : 0; + + var (sortedSetBytes, sortedSetKeys) = await SumMemoryUsageAsync( + server.Keys(dbIndex, "lb:*", pageSize: 1000), + db, + key => !key.EndsWith(":temp", StringComparison.Ordinal), + ct); + var (memberHashBytes, memberHashKeys) = await SumMemoryUsageAsync( + server.Keys(dbIndex, "member:*", pageSize: 1000), + db, + _ => true, + ct); + var (minScoreBytes, minScoreKeys) = await SumMemoryUsageAsync( + server.Keys(dbIndex, "lb-min:*", pageSize: 1000), + db, + _ => true, + ct); + + var totalBytes = sortedSetBytes + memberHashBytes + minScoreBytes; + Interlocked.Exchange(ref _sortedSetBytes, sortedSetBytes); + Interlocked.Exchange(ref _memberHashBytes, memberHashBytes); + Interlocked.Exchange(ref _minScoreBytes, minScoreBytes); + Interlocked.Exchange(ref _totalBytes, totalBytes); + Interlocked.Exchange(ref _sortedSetKeys, sortedSetKeys); + Interlocked.Exchange(ref _memberHashKeys, memberHashKeys); + Interlocked.Exchange(ref _minScoreKeys, minScoreKeys); + Interlocked.Exchange(ref _lastRefreshUnixSeconds, DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + Interlocked.Exchange(ref _refreshSuccessful, 1); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) { + // Graceful shutdown + } + catch (RedisServerException ex) { + Interlocked.Exchange(ref _refreshSuccessful, 0); + _logger.LogWarning(ex, + "Failed to sample leaderboard Redis memory. Ensure the MEMORY command is enabled on Redis."); + } + catch (Exception ex) { + Interlocked.Exchange(ref _refreshSuccessful, 0); + _logger.LogWarning(ex, "Failed to sample leaderboard Redis memory usage."); + } + } + + private static async Task<(long totalBytes, long keyCount)> SumMemoryUsageAsync( + IEnumerable keys, + IDatabase db, + Func includePredicate, + CancellationToken ct) { + long totalBytes = 0; + long keyCount = 0; + var tasks = new List>(MemoryUsageBatchSize); + + foreach (var key in keys) { + ct.ThrowIfCancellationRequested(); + var keyString = key.ToString(); + if (!includePredicate(keyString)) continue; + + tasks.Add(db.ExecuteAsync("MEMORY", "USAGE", key)); + if (tasks.Count < MemoryUsageBatchSize) continue; + + var (chunkBytes, chunkCount) = await DrainBatchAsync(tasks); + totalBytes += chunkBytes; + keyCount += chunkCount; + } + + if (tasks.Count > 0) { + var (chunkBytes, chunkCount) = await DrainBatchAsync(tasks); + totalBytes += chunkBytes; + keyCount += chunkCount; + } + + return (totalBytes, keyCount); + } + + private static async Task<(long totalBytes, long keyCount)> DrainBatchAsync(List> tasks) { + var results = await Task.WhenAll(tasks); + tasks.Clear(); + + long totalBytes = 0; + long keyCount = 0; + foreach (var result in results) { + if (result.IsNull) continue; + if (!long.TryParse(result.ToString(), out var bytes) || bytes < 0) continue; + totalBytes += bytes; + keyCount++; + } + + return (totalBytes, keyCount); + } +} diff --git a/EliteAPI/Features/Leaderboards/Services/LeaderboardRedisSyncService.cs b/EliteAPI/Features/Leaderboards/Services/LeaderboardRedisSyncService.cs index 81b305a6..586decaa 100644 --- a/EliteAPI/Features/Leaderboards/Services/LeaderboardRedisSyncService.cs +++ b/EliteAPI/Features/Leaderboards/Services/LeaderboardRedisSyncService.cs @@ -9,6 +9,7 @@ namespace EliteAPI.Features.Leaderboards.Services; public class LeaderboardRedisSyncService( IServiceProvider serviceProvider, IConnectionMultiplexer redis, + LeaderboardRedisMemoryMetricsService memoryMetrics, ILogger logger, IOptions settings) : BackgroundService { @@ -194,5 +195,8 @@ private async Task UpdateRequestedLeaderboards(CancellationToken ct) // Give database a small break between leaderboards await Task.Delay(1000, ct); } + + // Refresh Redis leaderboard memory gauges right after rebuild work. + await memoryMetrics.RefreshAsync(ct); } } diff --git a/EliteAPI/Utilities/ServiceExtensions.cs b/EliteAPI/Utilities/ServiceExtensions.cs index 357b96e0..ca029aab 100644 --- a/EliteAPI/Utilities/ServiceExtensions.cs +++ b/EliteAPI/Utilities/ServiceExtensions.cs @@ -44,6 +44,7 @@ public static IServiceCollection AddEliteServices(this IServiceCollection servic services.AddHostedService(); services.AddHostedService(); services.AddHostedService(); + services.AddSingleton(); services.AddSingleton(); services.AddHostedService(sp => sp.GetRequiredService()); @@ -260,4 +261,4 @@ public static class CachePolicy { public const string NoCache = "NoCache"; public const string Hours = "Hours"; -} \ No newline at end of file +}