From edcdae068c275a1955ff5e5691f5a23f6cc4e30d Mon Sep 17 00:00:00 2001 From: ptlthg <24925519+ptlthg@users.noreply.github.com> Date: Sun, 21 Sep 2025 22:55:42 -0400 Subject: [PATCH 1/4] Start working on skull rendering --- .../Services/MinecraftHeadRenderer.cs | 342 ++++++++++++++++++ backend/RepoAPI/RepoAPI.csproj | 2 + 2 files changed, 344 insertions(+) create mode 100644 backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs diff --git a/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs b/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs new file mode 100644 index 0000000..15a4916 --- /dev/null +++ b/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs @@ -0,0 +1,342 @@ +using System.Numerics; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing; + +[RegisterService(LifeTime.Scoped)] +public class MinecraftHeadRenderer(HttpClient httpClient) +{ + public record RenderOptions( + string SkinUrl, + int Size, + float YawInDegrees, + float PitchInDegrees, + float RollInDegrees, + bool ShowOverlay = true); + + public enum IsometricSide { Left, Right } + public record IsometricRenderOptions(string SkinUrl, int Size, IsometricSide Side = IsometricSide.Right, bool ShowOverlay = true); + + private static readonly Dictionary BaseMappings = new() + { + { Face.Right, new Rectangle(0, 8, 8, 8) }, + { Face.Front, new Rectangle(8, 8, 8, 8) }, + { Face.Left, new Rectangle(16, 8, 8, 8) }, + { Face.Back, new Rectangle(24, 8, 8, 8) }, + { Face.Top, new Rectangle(8, 0, 8, 8) }, + { Face.Bottom, new Rectangle(16, 0, 8, 8) } + }; + + private static readonly Dictionary OverlayMappings = new() + { + { Face.Right, new Rectangle(32, 8, 8, 8) }, + { Face.Front, new Rectangle(40, 8, 8, 8) }, + { Face.Left, new Rectangle(48, 8, 8, 8) }, + { Face.Back, new Rectangle(56, 8, 8, 8) }, + { Face.Top, new Rectangle(40, 0, 8, 8) }, + { Face.Bottom, new Rectangle(48, 0, 8, 8) } + }; + + public Task> RenderIsometricHeadAsync(IsometricRenderOptions options, CancellationToken ct = default) + { + // Isometric view: showing front, right, and top faces (or left if specified) + const float isometricRightYaw = -135f; + const float isometricLeftYaw = 45f; + const float isometricPitch = 30f; + const float isometricRoll = 0f; + + var fullOptions = new RenderOptions( + options.SkinUrl, + options.Size, + options.Side == IsometricSide.Right ? isometricRightYaw : isometricLeftYaw, + isometricPitch, + isometricRoll, + options.ShowOverlay + ); + + return RenderHeadAsync(fullOptions, ct); + } + + public async Task> RenderHeadAsync(RenderOptions options, CancellationToken ct = default) + { + using var skin = await Image.LoadAsync(await httpClient.GetStreamAsync(options.SkinUrl, ct), ct); + return RenderHead(options, skin); + } + + public static Image RenderHead(RenderOptions options, Image skin) + { + // Define cube vertices (unit cube centered at origin) + var vertices = new Vector3[] + { + // Back face vertices (z = -0.5) + new(-0.5f, -0.5f, -0.5f), // 0: bottom-left-back + new(0.5f, -0.5f, -0.5f), // 1: bottom-right-back + new(0.5f, 0.5f, -0.5f), // 2: top-right-back + new(-0.5f, 0.5f, -0.5f), // 3: top-left-back + + // Front face vertices (z = 0.5) + new(-0.5f, -0.5f, 0.5f), // 4: bottom-left-front + new(0.5f, -0.5f, 0.5f), // 5: bottom-right-front + new(0.5f, 0.5f, 0.5f), // 6: top-right-front + new(-0.5f, 0.5f, 0.5f) // 7: top-left-front + }; + + var standardUvMap = new Vector2[] + { + new(1, 0), new(0, 0), new(0, 1), new(1, 1) + }; + // I don't know why this face needs to be flipped, but it does + var backFaceUvMap = new Vector2[] { new(1, 1), new(0, 1), new(0, 0), new(1, 0) }; + + // Define faces with correct winding order and UV mappings + var faceDefinitions = new[] + { + // Front face (+Z) + new FaceData(Face.Front, [vertices[7], vertices[6], vertices[5], vertices[4]], standardUvMap), + // Back face (-Z) + new FaceData(Face.Back, [vertices[0], vertices[1], vertices[2], vertices[3]], backFaceUvMap), + // Right face (+X) + new FaceData(Face.Right, [vertices[6], vertices[2], vertices[1], vertices[5]], standardUvMap), + // Left face (-X) + new FaceData(Face.Left, [vertices[3], vertices[7], vertices[4], vertices[0]], standardUvMap), + // Top face (+Y) + new FaceData(Face.Top, [vertices[3], vertices[2], vertices[6], vertices[7]], standardUvMap), + // Bottom face (-Y) + new FaceData(Face.Bottom, [vertices[4], vertices[5], vertices[1], vertices[0]], standardUvMap) + }; + + // Create rotation matrices + const float deg2Rad = MathF.PI / 180f; + var transform = CreateRotationMatrix( + options.YawInDegrees * deg2Rad, + options.PitchInDegrees * deg2Rad, + options.RollInDegrees * deg2Rad + ); + + // Process base layer + var visibleTriangles = ProcessFaces(faceDefinitions, transform, skin, false); + + // Process overlay layer if enabled + if (options.ShowOverlay) + { + var overlayTransform = Matrix4x4.CreateScale(1.125f) * transform; + visibleTriangles.AddRange(ProcessFaces(faceDefinitions, overlayTransform, skin, true)); + } + + // Sort triangles by depth (back to front) + visibleTriangles.Sort((a, b) => b.Depth.CompareTo(a.Depth)); + + // Create output image + var canvas = new Image(options.Size, options.Size, Color.Transparent); + var scale = options.Size / 1.75f; + var offset = new Vector2(options.Size / 2f); + + // Render triangles + foreach (var tri in visibleTriangles) + { + var p1 = ProjectToScreen(tri.V1, scale, offset); + var p2 = ProjectToScreen(tri.V2, scale, offset); + var p3 = ProjectToScreen(tri.V3, scale, offset); + + RasterizeTriangle(canvas, p1, p2, p3, tri.T1, tri.T2, tri.T3, tri.Texture); + } + + return canvas; + } + + private static Matrix4x4 CreateRotationMatrix(float yaw, float pitch, float roll) + { + // Apply rotations in Y-X-Z order for intuitive control + var cosY = MathF.Cos(yaw); + var sinY = MathF.Sin(yaw); + var cosP = MathF.Cos(pitch); + var sinP = MathF.Sin(pitch); + var cosR = MathF.Cos(roll); + var sinR = MathF.Sin(roll); + + // Combined rotation matrix (Y * X * Z) + return new Matrix4x4( + cosY * cosR + sinY * sinP * sinR, -cosY * sinR + sinY * sinP * cosR, sinY * cosP, 0, + cosP * sinR, cosP * cosR, -sinP, 0, + -sinY * cosR + cosY * sinP * sinR, sinY * sinR + cosY * sinP * cosR, cosY * cosP, 0, + 0, 0, 0, 1 + ); + } + + private static Vector2 ProjectToScreen(Vector3 point, float scale, Vector2 offset) + { + // Orthographic projection to 2D screen space (flip Y for screen coordinates) + return new Vector2(point.X * scale + offset.X, -point.Y * scale + offset.Y); + + // This value (from 0.0 to 1.0) controls the strength of the perspective effect. + // 0.0 = Fully orthographic (flat, no distortion) + // 1.0 = Full perspective + // const float perspectiveAmount = 0.2f; + // + // // Calculate the full perspective projection + // const float cameraDistance = 10.0f; // This can stay fixed + // const float focalLength = 10.0f; + // var perspectiveFactor = focalLength / (cameraDistance - point.Z); + // var perspX = point.X * perspectiveFactor; + // var perspY = point.Y * perspectiveFactor; + + // // Orthographic projection (no perspective) + // var orthoX = point.X; + // var orthoY = point.Y; + // + // var finalX = orthoX + (perspX - orthoX) * perspectiveAmount; + // var finalY = orthoY + (perspY - orthoY) * perspectiveAmount; + + // return new Vector2( + // finalX * scale + offset.X, + // -finalY * scale + offset.Y + // ); + } + + private static List ProcessFaces(FaceData[] faces, Matrix4x4 transform, Image skin, + bool isOverlay) + { + var triangles = new List(); + var mappings = isOverlay ? OverlayMappings : BaseMappings; + + foreach (var face in faces) + { + var texRect = mappings[face.FaceType]; + + // Extract texture for this face + using var faceTexture = skin.Clone(ctx => ctx.Crop(texRect)); + + // Transform vertices + var transformed = new Vector3[4]; + for (var i = 0; i < 4; i++) transformed[i] = Vector3.Transform(face.Vertices[i], transform); + + if (!isOverlay) + { + // Calculate face normal for backface culling (optional) + var normal = Vector3.Cross( + transformed[1] - transformed[0], + transformed[2] - transformed[0] + ); + + // Skip back-facing triangles (keep this commented if you want to see all faces) + if (normal.Z < 0) continue; + } + + // Calculate average depth for sorting + var depth = (transformed[0].Z + transformed[1].Z + transformed[2].Z + transformed[3].Z) / 4f; + + // Create two triangles for the quad + triangles.Add(new VisibleTriangle( + transformed[0], transformed[1], transformed[2], + face.UvMap[0], face.UvMap[1], face.UvMap[2], + faceTexture.Clone(), depth + )); + + triangles.Add(new VisibleTriangle( + transformed[0], transformed[2], transformed[3], + face.UvMap[0], face.UvMap[2], face.UvMap[3], + faceTexture.Clone(), depth + )); + } + + return triangles; + } + + private static void RasterizeTriangle( + Image canvas, + Vector2 p1, Vector2 p2, Vector2 p3, + Vector2 t1, Vector2 t2, Vector2 t3, + Image texture) + { + var area = (p2.X - p1.X) * (p3.Y - p1.Y) - (p3.X - p1.X) * (p2.Y - p1.Y); + + // If the area is negligible, the triangle is a line or a point. Skip it. + if (Math.Abs(area) < 0.01f) + { + texture.Dispose(); // Still need to dispose the cloned texture + return; + } + + // Calculate bounding box + var minX = (int)Math.Max(0, Math.Min(Math.Min(p1.X, p2.X), p3.X)); + var minY = (int)Math.Max(0, Math.Min(Math.Min(p1.Y, p2.Y), p3.Y)); + var maxX = (int)Math.Min(canvas.Width - 1, Math.Ceiling(Math.Max(Math.Max(p1.X, p2.X), p3.X))); + var maxY = (int)Math.Min(canvas.Height - 1, Math.Ceiling(Math.Max(Math.Max(p1.Y, p2.Y), p3.Y))); + + // Rasterize triangle + for (var y = minY; y <= maxY; y++) + for (var x = minX; x <= maxX; x++) + { + var point = new Vector2(x + 0.5f, y + 0.5f); + var bary = GetBarycentric(p1, p2, p3, point); + + // Check if point is inside triangle + const float epsilon = 1e-5f; + if (bary.X < -epsilon || bary.Y < -epsilon || bary.Z < -epsilon) continue; + + // Interpolate texture coordinates + var texCoord = t1 * bary.X + t2 * bary.Y + t3 * bary.Z; + + // Sample texture + var texX = (int)Math.Clamp(texCoord.X * texture.Width, 0, texture.Width - 1); + var texY = (int)Math.Clamp(texCoord.Y * texture.Height, 0, texture.Height - 1); + + var color = texture[texX, texY]; + if (color.A > 10) // Skip nearly transparent pixels + // Alpha blend if needed, or just overwrite + canvas[x, y] = color; + } + + texture.Dispose(); + } + + private static Vector3 GetBarycentric(Vector2 a, Vector2 b, Vector2 c, Vector2 p) + { + var v0 = b - a; + var v1 = c - a; + var v2 = p - a; + + var d00 = Vector2.Dot(v0, v0); + var d01 = Vector2.Dot(v0, v1); + var d11 = Vector2.Dot(v1, v1); + var d20 = Vector2.Dot(v2, v0); + var d21 = Vector2.Dot(v2, v1); + + var denom = d00 * d11 - d01 * d01; + if (Math.Abs(denom) < 1e-6f) return new Vector3(-1, -1, -1); + + var v = (d11 * d20 - d01 * d21) / denom; + var w = (d00 * d21 - d01 * d20) / denom; + var u = 1.0f - v - w; + + return new Vector3(u, v, w); + } + + private enum Face + { + Top, + Bottom, + Left, + Right, + Front, + Back + } + + private record FaceData(Face FaceType, Vector3[] Vertices, Vector2[] UvMap); + + private record VisibleTriangle( + Vector3 V1, + Vector3 V2, + Vector3 V3, + Vector2 T1, + Vector2 T2, + Vector2 T3, + Image Texture, + float Depth) : IDisposable + { + public void Dispose() + { + Texture?.Dispose(); + } + } +} \ No newline at end of file diff --git a/backend/RepoAPI/RepoAPI.csproj b/backend/RepoAPI/RepoAPI.csproj index c7a668d..5e58cab 100644 --- a/backend/RepoAPI/RepoAPI.csproj +++ b/backend/RepoAPI/RepoAPI.csproj @@ -44,6 +44,8 @@ + + From 610b0f4b70da5f5a79545d0724120161ae208fd4 Mon Sep 17 00:00:00 2001 From: ptlthg <24925519+ptlthg@users.noreply.github.com> Date: Mon, 22 Sep 2025 19:11:40 -0400 Subject: [PATCH 2/4] Optimize head rendering, went from ~5ms to ~1ms per head --- .../Services/MinecraftHeadRenderer.cs | 282 +++++++++--------- 1 file changed, 139 insertions(+), 143 deletions(-) diff --git a/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs b/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs index 15a4916..56ca8ba 100644 --- a/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs +++ b/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs @@ -1,7 +1,7 @@ using System.Numerics; using SixLabors.ImageSharp; +using SixLabors.ImageSharp.Advanced; using SixLabors.ImageSharp.PixelFormats; -using SixLabors.ImageSharp.Processing; [RegisterService(LifeTime.Scoped)] public class MinecraftHeadRenderer(HttpClient httpClient) @@ -12,6 +12,7 @@ public record RenderOptions( float YawInDegrees, float PitchInDegrees, float RollInDegrees, + float PerspectiveAmount = 0f, bool ShowOverlay = true); public enum IsometricSide { Left, Right } @@ -36,6 +37,46 @@ public record IsometricRenderOptions(string SkinUrl, int Size, IsometricSide Sid { Face.Top, new Rectangle(40, 0, 8, 8) }, { Face.Bottom, new Rectangle(48, 0, 8, 8) } }; + + // Define cube vertices (unit cube centered at origin) + private static readonly Vector3[] Vertices = + [ + // Back face vertices (z = -0.5) + new(-0.5f, -0.5f, -0.5f), // 0: bottom-left-back + new(0.5f, -0.5f, -0.5f), // 1: bottom-right-back + new(0.5f, 0.5f, -0.5f), // 2: top-right-back + new(-0.5f, 0.5f, -0.5f), // 3: top-left-back + + // Front face vertices (z = 0.5) + new(-0.5f, -0.5f, 0.5f), // 4: bottom-left-front + new(0.5f, -0.5f, 0.5f), // 5: bottom-right-front + new(0.5f, 0.5f, 0.5f), // 6: top-right-front + new(-0.5f, 0.5f, 0.5f) // 7: top-left-front + ]; + + private static readonly Vector2[] StandardUvMap = + [ + new(1, 0), new(0, 0), new(0, 1), new(1, 1) + ]; + // I don't know why this face needs to be flipped, but it does + private static readonly Vector2[] BackFaceUvMap = [new(1, 1), new(0, 1), new(0, 0), new(1, 0)]; + + // Define faces with correct winding order and UV mappings + private static readonly FaceData[] FaceDefinitions = + [ + // Front face (+Z) + new FaceData(Face.Front, [Vertices[7], Vertices[6], Vertices[5], Vertices[4]], StandardUvMap), + // Back face (-Z) + new FaceData(Face.Back, [Vertices[0], Vertices[1], Vertices[2], Vertices[3]], BackFaceUvMap), + // Right face (+X) + new FaceData(Face.Right, [Vertices[6], Vertices[2], Vertices[1], Vertices[5]], StandardUvMap), + // Left face (-X) + new FaceData(Face.Left, [Vertices[3], Vertices[7], Vertices[4], Vertices[0]], StandardUvMap), + // Top face (+Y) + new FaceData(Face.Top, [Vertices[3], Vertices[2], Vertices[6], Vertices[7]], StandardUvMap), + // Bottom face (-Y) + new FaceData(Face.Bottom, [Vertices[4], Vertices[5], Vertices[1], Vertices[0]], StandardUvMap) + ]; public Task> RenderIsometricHeadAsync(IsometricRenderOptions options, CancellationToken ct = default) { @@ -51,7 +92,7 @@ public Task> RenderIsometricHeadAsync(IsometricRenderOptions optio options.Side == IsometricSide.Right ? isometricRightYaw : isometricLeftYaw, isometricPitch, isometricRoll, - options.ShowOverlay + ShowOverlay: options.ShowOverlay ); return RenderHeadAsync(fullOptions, ct); @@ -65,46 +106,6 @@ public async Task> RenderHeadAsync(RenderOptions options, Cancella public static Image RenderHead(RenderOptions options, Image skin) { - // Define cube vertices (unit cube centered at origin) - var vertices = new Vector3[] - { - // Back face vertices (z = -0.5) - new(-0.5f, -0.5f, -0.5f), // 0: bottom-left-back - new(0.5f, -0.5f, -0.5f), // 1: bottom-right-back - new(0.5f, 0.5f, -0.5f), // 2: top-right-back - new(-0.5f, 0.5f, -0.5f), // 3: top-left-back - - // Front face vertices (z = 0.5) - new(-0.5f, -0.5f, 0.5f), // 4: bottom-left-front - new(0.5f, -0.5f, 0.5f), // 5: bottom-right-front - new(0.5f, 0.5f, 0.5f), // 6: top-right-front - new(-0.5f, 0.5f, 0.5f) // 7: top-left-front - }; - - var standardUvMap = new Vector2[] - { - new(1, 0), new(0, 0), new(0, 1), new(1, 1) - }; - // I don't know why this face needs to be flipped, but it does - var backFaceUvMap = new Vector2[] { new(1, 1), new(0, 1), new(0, 0), new(1, 0) }; - - // Define faces with correct winding order and UV mappings - var faceDefinitions = new[] - { - // Front face (+Z) - new FaceData(Face.Front, [vertices[7], vertices[6], vertices[5], vertices[4]], standardUvMap), - // Back face (-Z) - new FaceData(Face.Back, [vertices[0], vertices[1], vertices[2], vertices[3]], backFaceUvMap), - // Right face (+X) - new FaceData(Face.Right, [vertices[6], vertices[2], vertices[1], vertices[5]], standardUvMap), - // Left face (-X) - new FaceData(Face.Left, [vertices[3], vertices[7], vertices[4], vertices[0]], standardUvMap), - // Top face (+Y) - new FaceData(Face.Top, [vertices[3], vertices[2], vertices[6], vertices[7]], standardUvMap), - // Bottom face (-Y) - new FaceData(Face.Bottom, [vertices[4], vertices[5], vertices[1], vertices[0]], standardUvMap) - }; - // Create rotation matrices const float deg2Rad = MathF.PI / 180f; var transform = CreateRotationMatrix( @@ -114,13 +115,13 @@ public static Image RenderHead(RenderOptions options, Image skin ); // Process base layer - var visibleTriangles = ProcessFaces(faceDefinitions, transform, skin, false); + var visibleTriangles = ProcessFaces(FaceDefinitions, transform, skin, false); // Process overlay layer if enabled if (options.ShowOverlay) { var overlayTransform = Matrix4x4.CreateScale(1.125f) * transform; - visibleTriangles.AddRange(ProcessFaces(faceDefinitions, overlayTransform, skin, true)); + visibleTriangles.AddRange(ProcessFaces(FaceDefinitions, overlayTransform, skin, true)); } // Sort triangles by depth (back to front) @@ -134,11 +135,11 @@ public static Image RenderHead(RenderOptions options, Image skin // Render triangles foreach (var tri in visibleTriangles) { - var p1 = ProjectToScreen(tri.V1, scale, offset); - var p2 = ProjectToScreen(tri.V2, scale, offset); - var p3 = ProjectToScreen(tri.V3, scale, offset); + var p1 = ProjectToScreen(tri.V1, scale, offset, options.PerspectiveAmount); + var p2 = ProjectToScreen(tri.V2, scale, offset, options.PerspectiveAmount); + var p3 = ProjectToScreen(tri.V3, scale, offset, options.PerspectiveAmount); - RasterizeTriangle(canvas, p1, p2, p3, tri.T1, tri.T2, tri.T3, tri.Texture); + RasterizeTriangle(canvas, p1, p2, p3, tri.T1, tri.T2, tri.T3, skin, tri.TextureRect); } return canvas; @@ -146,7 +147,7 @@ public static Image RenderHead(RenderOptions options, Image skin private static Matrix4x4 CreateRotationMatrix(float yaw, float pitch, float roll) { - // Apply rotations in Y-X-Z order for intuitive control + // Apply rotations in Y-X-Z order var cosY = MathF.Cos(yaw); var sinY = MathF.Sin(yaw); var cosP = MathF.Cos(pitch); @@ -163,62 +164,59 @@ private static Matrix4x4 CreateRotationMatrix(float yaw, float pitch, float roll ); } - private static Vector2 ProjectToScreen(Vector3 point, float scale, Vector2 offset) + private static Vector2 ProjectToScreen(Vector3 point, float scale, Vector2 offset, float perspectiveAmount = 0f) { // Orthographic projection to 2D screen space (flip Y for screen coordinates) - return new Vector2(point.X * scale + offset.X, -point.Y * scale + offset.Y); + if (perspectiveAmount <= 0.01f) { + return new Vector2(point.X * scale + offset.X, -point.Y * scale + offset.Y); + } + + // Calculate the full perspective projection + const float cameraDistance = 10.0f; + const float focalLength = 10.0f; + var perspectiveFactor = focalLength / (cameraDistance - point.Z); + var perspX = point.X * perspectiveFactor; + var perspY = point.Y * perspectiveFactor; + + // Orthographic projection (no perspective) + var orthoX = point.X; + var orthoY = point.Y; - // This value (from 0.0 to 1.0) controls the strength of the perspective effect. - // 0.0 = Fully orthographic (flat, no distortion) - // 1.0 = Full perspective - // const float perspectiveAmount = 0.2f; - // - // // Calculate the full perspective projection - // const float cameraDistance = 10.0f; // This can stay fixed - // const float focalLength = 10.0f; - // var perspectiveFactor = focalLength / (cameraDistance - point.Z); - // var perspX = point.X * perspectiveFactor; - // var perspY = point.Y * perspectiveFactor; - - // // Orthographic projection (no perspective) - // var orthoX = point.X; - // var orthoY = point.Y; - // - // var finalX = orthoX + (perspX - orthoX) * perspectiveAmount; - // var finalY = orthoY + (perspY - orthoY) * perspectiveAmount; - - // return new Vector2( - // finalX * scale + offset.X, - // -finalY * scale + offset.Y - // ); + var finalX = orthoX + (perspX - orthoX) * perspectiveAmount; + var finalY = orthoY + (perspY - orthoY) * perspectiveAmount; + + return new Vector2( + finalX * scale + offset.X, + -finalY * scale + offset.Y + ); } private static List ProcessFaces(FaceData[] faces, Matrix4x4 transform, Image skin, bool isOverlay) { - var triangles = new List(); + var triangles = new List(faces.Length * 2); var mappings = isOverlay ? OverlayMappings : BaseMappings; foreach (var face in faces) { - var texRect = mappings[face.FaceType]; - // Extract texture for this face - using var faceTexture = skin.Clone(ctx => ctx.Crop(texRect)); - - // Transform vertices + var texRect = mappings[face.FaceType]; var transformed = new Vector3[4]; - for (var i = 0; i < 4; i++) transformed[i] = Vector3.Transform(face.Vertices[i], transform); - - if (!isOverlay) + for (var i = 0; i < 4; i++) { - // Calculate face normal for backface culling (optional) + transformed[i] = Vector3.Transform(face.Vertices[i], transform); + } + + // Backface culling for non-overlay faces + // Overlay faces are always drawn to ensure visibility around edges + if (!isOverlay) { + // Calculate face normal for backface culling var normal = Vector3.Cross( transformed[1] - transformed[0], transformed[2] - transformed[0] ); - // Skip back-facing triangles (keep this commented if you want to see all faces) + // Skip back-facing triangles if (normal.Z < 0) continue; } @@ -229,13 +227,13 @@ private static List ProcessFaces(FaceData[] faces, Matrix4x4 tr triangles.Add(new VisibleTriangle( transformed[0], transformed[1], transformed[2], face.UvMap[0], face.UvMap[1], face.UvMap[2], - faceTexture.Clone(), depth + texRect, depth )); triangles.Add(new VisibleTriangle( transformed[0], transformed[2], transformed[3], face.UvMap[0], face.UvMap[2], face.UvMap[3], - faceTexture.Clone(), depth + texRect, depth )); } @@ -246,17 +244,24 @@ private static void RasterizeTriangle( Image canvas, Vector2 p1, Vector2 p2, Vector2 p3, Vector2 t1, Vector2 t2, Vector2 t3, - Image texture) + Image skin, Rectangle textureRect) { var area = (p2.X - p1.X) * (p3.Y - p1.Y) - (p3.X - p1.X) * (p2.Y - p1.Y); + if (Math.Abs(area) < 0.01f) return; // Degenerate triangle + + // Pre-calculate values that are constant for every pixel in the triangle. + var v0 = p2 - p1; + var v1 = p3 - p1; + var d00 = Vector2.Dot(v0, v0); + var d01 = Vector2.Dot(v0, v1); + var d11 = Vector2.Dot(v1, v1); + var denom = d00 * d11 - d01 * d01; - // If the area is negligible, the triangle is a line or a point. Skip it. - if (Math.Abs(area) < 0.01f) - { - texture.Dispose(); // Still need to dispose the cloned texture - return; - } - + // If the denominator is zero, the triangle is degenerate (a line or point). + if (Math.Abs(denom) < 1e-6f) return; + + var baryData = new BarycentricData(v0, v1, d00, d01, d11, denom); + // Calculate bounding box var minX = (int)Math.Max(0, Math.Min(Math.Min(p1.X, p2.X), p3.X)); var minY = (int)Math.Max(0, Math.Min(Math.Min(p1.Y, p2.Y), p3.Y)); @@ -264,54 +269,49 @@ private static void RasterizeTriangle( var maxY = (int)Math.Min(canvas.Height - 1, Math.Ceiling(Math.Max(Math.Max(p1.Y, p2.Y), p3.Y))); // Rasterize triangle - for (var y = minY; y <= maxY; y++) - for (var x = minX; x <= maxX; x++) + Parallel.For((long) minY, maxY + 1, y => { - var point = new Vector2(x + 0.5f, y + 0.5f); - var bary = GetBarycentric(p1, p2, p3, point); - - // Check if point is inside triangle - const float epsilon = 1e-5f; - if (bary.X < -epsilon || bary.Y < -epsilon || bary.Z < -epsilon) continue; - - // Interpolate texture coordinates - var texCoord = t1 * bary.X + t2 * bary.Y + t3 * bary.Z; - - // Sample texture - var texX = (int)Math.Clamp(texCoord.X * texture.Width, 0, texture.Width - 1); - var texY = (int)Math.Clamp(texCoord.Y * texture.Height, 0, texture.Height - 1); - - var color = texture[texX, texY]; - if (color.A > 10) // Skip nearly transparent pixels - // Alpha blend if needed, or just overwrite - canvas[x, y] = color; - } + // Get a span for the current row for fast, direct memory access. + // Dangerous, but should be fine as canvas's lifetime is not at risk here. + var canvasRow = canvas.DangerousGetPixelRowMemory((int) y).Span; - texture.Dispose(); + for (var x = minX; x <= maxX; x++) + { + var point = new Vector2(x + 0.5f, y + 0.5f); + var bary = GetBarycentric(p1, point, in baryData); + + const float epsilon = 1e-5f; + if (bary.X < -epsilon || bary.Y < -epsilon || bary.Z < -epsilon) continue; + + var texCoord = t1 * bary.X + t2 * bary.Y + t3 * bary.Z; + + var texX = (int)Math.Clamp(texCoord.X * textureRect.Width, 0, textureRect.Width - 1); + var texY = (int)Math.Clamp(texCoord.Y * textureRect.Height, 0, textureRect.Height - 1); + + var color = skin[textureRect.X + texX, textureRect.Y + texY]; + + if (color.A > 10) + { + // Write directly to the canvas row's memory via the span. + canvasRow[x] = color; + } + } + }); } - private static Vector3 GetBarycentric(Vector2 a, Vector2 b, Vector2 c, Vector2 p) + private static Vector3 GetBarycentric(Vector2 p1, Vector2 p, in BarycentricData data) { - var v0 = b - a; - var v1 = c - a; - var v2 = p - a; - - var d00 = Vector2.Dot(v0, v0); - var d01 = Vector2.Dot(v0, v1); - var d11 = Vector2.Dot(v1, v1); - var d20 = Vector2.Dot(v2, v0); - var d21 = Vector2.Dot(v2, v1); - - var denom = d00 * d11 - d01 * d01; - if (Math.Abs(denom) < 1e-6f) return new Vector3(-1, -1, -1); - - var v = (d11 * d20 - d01 * d21) / denom; - var w = (d00 * d21 - d01 * d20) / denom; + var v2 = p - p1; + var d20 = Vector2.Dot(v2, data.V0); + var d21 = Vector2.Dot(v2, data.V1); + + var v = (data.D11 * d20 - data.D01 * d21) / data.Denom; + var w = (data.D00 * d21 - data.D01 * d20) / data.Denom; var u = 1.0f - v - w; - + return new Vector3(u, v, w); } - + private enum Face { Top, @@ -331,12 +331,8 @@ private record VisibleTriangle( Vector2 T1, Vector2 T2, Vector2 T3, - Image Texture, - float Depth) : IDisposable - { - public void Dispose() - { - Texture?.Dispose(); - } - } + Rectangle TextureRect, + float Depth); + + private readonly record struct BarycentricData(Vector2 V0, Vector2 V1, float D00, float D01, float D11, float Denom); } \ No newline at end of file From f8ca07298b72962e1fa8f6e20cfe0497683b3b2b Mon Sep 17 00:00:00 2001 From: ptlthg <24925519+ptlthg@users.noreply.github.com> Date: Mon, 22 Sep 2025 20:14:04 -0400 Subject: [PATCH 3/4] Very small improvements and fix bottom texture --- .../Services/MinecraftHeadRenderer.cs | 94 ++++++++++--------- 1 file changed, 51 insertions(+), 43 deletions(-) diff --git a/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs b/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs index 56ca8ba..0e85071 100644 --- a/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs +++ b/backend/RepoAPI/Features/Rendering/Services/MinecraftHeadRenderer.cs @@ -1,4 +1,5 @@ using System.Numerics; +using System.Runtime.CompilerServices; using SixLabors.ImageSharp; using SixLabors.ImageSharp.Advanced; using SixLabors.ImageSharp.PixelFormats; @@ -58,8 +59,8 @@ public record IsometricRenderOptions(string SkinUrl, int Size, IsometricSide Sid [ new(1, 0), new(0, 0), new(0, 1), new(1, 1) ]; - // I don't know why this face needs to be flipped, but it does private static readonly Vector2[] BackFaceUvMap = [new(1, 1), new(0, 1), new(0, 0), new(1, 0)]; + private static readonly Vector2[] BottomFaceUvMap = [new(0, 1), new(1, 1), new(1, 0), new(0, 0)]; // Define faces with correct winding order and UV mappings private static readonly FaceData[] FaceDefinitions = @@ -75,10 +76,10 @@ public record IsometricRenderOptions(string SkinUrl, int Size, IsometricSide Sid // Top face (+Y) new FaceData(Face.Top, [Vertices[3], Vertices[2], Vertices[6], Vertices[7]], StandardUvMap), // Bottom face (-Y) - new FaceData(Face.Bottom, [Vertices[4], Vertices[5], Vertices[1], Vertices[0]], StandardUvMap) + new FaceData(Face.Bottom, [Vertices[4], Vertices[5], Vertices[1], Vertices[0]], BottomFaceUvMap) ]; - public Task> RenderIsometricHeadAsync(IsometricRenderOptions options, CancellationToken ct = default) + public async Task> RenderIsometricHeadAsync(IsometricRenderOptions options, CancellationToken ct = default) { // Isometric view: showing front, right, and top faces (or left if specified) const float isometricRightYaw = -135f; @@ -95,7 +96,7 @@ public Task> RenderIsometricHeadAsync(IsometricRenderOptions optio ShowOverlay: options.ShowOverlay ); - return RenderHeadAsync(fullOptions, ct); + return await RenderHeadAsync(fullOptions, ct); } public async Task> RenderHeadAsync(RenderOptions options, CancellationToken ct = default) @@ -113,15 +114,18 @@ public static Image RenderHead(RenderOptions options, Image skin options.PitchInDegrees * deg2Rad, options.RollInDegrees * deg2Rad ); + + var initialCapacity = options.ShowOverlay ? FaceDefinitions.Length * 4 : FaceDefinitions.Length * 2; + var visibleTriangles = new List(initialCapacity); // Process base layer - var visibleTriangles = ProcessFaces(FaceDefinitions, transform, skin, false); + ProcessFaces(FaceDefinitions, transform, false, visibleTriangles); // Process overlay layer if enabled if (options.ShowOverlay) { var overlayTransform = Matrix4x4.CreateScale(1.125f) * transform; - visibleTriangles.AddRange(ProcessFaces(FaceDefinitions, overlayTransform, skin, true)); + ProcessFaces(FaceDefinitions, overlayTransform, true, visibleTriangles); } // Sort triangles by depth (back to front) @@ -132,19 +136,24 @@ public static Image RenderHead(RenderOptions options, Image skin var scale = options.Size / 1.75f; var offset = new Vector2(options.Size / 2f); + // Pre-calculate perspective parameters if needed + PerspectiveParams? perspectiveParams = options.PerspectiveAmount > 0.01f ? + new PerspectiveParams(options.PerspectiveAmount, 10.0f, 10.0f) : + null; + // Render triangles foreach (var tri in visibleTriangles) { - var p1 = ProjectToScreen(tri.V1, scale, offset, options.PerspectiveAmount); - var p2 = ProjectToScreen(tri.V2, scale, offset, options.PerspectiveAmount); - var p3 = ProjectToScreen(tri.V3, scale, offset, options.PerspectiveAmount); + var p1 = ProjectToScreen(tri.V1, scale, offset, perspectiveParams); + var p2 = ProjectToScreen(tri.V2, scale, offset, perspectiveParams); + var p3 = ProjectToScreen(tri.V3, scale, offset, perspectiveParams); RasterizeTriangle(canvas, p1, p2, p3, tri.T1, tri.T2, tri.T3, skin, tri.TextureRect); } return canvas; } - + private static Matrix4x4 CreateRotationMatrix(float yaw, float pitch, float roll) { // Apply rotations in Y-X-Z order @@ -163,18 +172,16 @@ private static Matrix4x4 CreateRotationMatrix(float yaw, float pitch, float roll 0, 0, 0, 1 ); } - - private static Vector2 ProjectToScreen(Vector3 point, float scale, Vector2 offset, float perspectiveAmount = 0f) + + private static Vector2 ProjectToScreen(Vector3 point, float scale, Vector2 offset, PerspectiveParams? perspectiveParams) { - // Orthographic projection to 2D screen space (flip Y for screen coordinates) - if (perspectiveAmount <= 0.01f) { + if (perspectiveParams == null) + { return new Vector2(point.X * scale + offset.X, -point.Y * scale + offset.Y); } // Calculate the full perspective projection - const float cameraDistance = 10.0f; - const float focalLength = 10.0f; - var perspectiveFactor = focalLength / (cameraDistance - point.Z); + var perspectiveFactor = perspectiveParams.Value.FocalLength / (perspectiveParams.Value.CameraDistance - point.Z); var perspX = point.X * perspectiveFactor; var perspY = point.Y * perspectiveFactor; @@ -182,8 +189,8 @@ private static Vector2 ProjectToScreen(Vector3 point, float scale, Vector2 offse var orthoX = point.X; var orthoY = point.Y; - var finalX = orthoX + (perspX - orthoX) * perspectiveAmount; - var finalY = orthoY + (perspY - orthoY) * perspectiveAmount; + var finalX = orthoX + (perspX - orthoX) * perspectiveParams.Value.Amount; + var finalY = orthoY + (perspY - orthoY) * perspectiveParams.Value.Amount; return new Vector2( finalX * scale + offset.X, @@ -191,37 +198,35 @@ private static Vector2 ProjectToScreen(Vector3 point, float scale, Vector2 offse ); } - private static List ProcessFaces(FaceData[] faces, Matrix4x4 transform, Image skin, - bool isOverlay) + private static void ProcessFaces(FaceData[] faces, Matrix4x4 transform, + bool isOverlay, List triangles) { - var triangles = new List(faces.Length * 2); var mappings = isOverlay ? OverlayMappings : BaseMappings; + Span transformed = stackalloc Vector3[4]; foreach (var face in faces) { // Extract texture for this face var texRect = mappings[face.FaceType]; - var transformed = new Vector3[4]; + for (var i = 0; i < 4; i++) { transformed[i] = Vector3.Transform(face.Vertices[i], transform); } // Backface culling for non-overlay faces - // Overlay faces are always drawn to ensure visibility around edges if (!isOverlay) { // Calculate face normal for backface culling - var normal = Vector3.Cross( - transformed[1] - transformed[0], - transformed[2] - transformed[0] - ); + var v1 = transformed[1] - transformed[0]; + var v2 = transformed[2] - transformed[0]; + var normal = Vector3.Cross(v1, v2); // Skip back-facing triangles if (normal.Z < 0) continue; } // Calculate average depth for sorting - var depth = (transformed[0].Z + transformed[1].Z + transformed[2].Z + transformed[3].Z) / 4f; + var depth = (transformed[0].Z + transformed[1].Z + transformed[2].Z + transformed[3].Z) * 0.25f; // Create two triangles for the quad triangles.Add(new VisibleTriangle( @@ -236,8 +241,6 @@ private static List ProcessFaces(FaceData[] faces, Matrix4x4 tr texRect, depth )); } - - return triangles; } private static void RasterizeTriangle( @@ -247,7 +250,7 @@ private static void RasterizeTriangle( Image skin, Rectangle textureRect) { var area = (p2.X - p1.X) * (p3.Y - p1.Y) - (p3.X - p1.X) * (p2.Y - p1.Y); - if (Math.Abs(area) < 0.01f) return; // Degenerate triangle + if (MathF.Abs(area) < 0.01f) return; // Degenerate triangle // Pre-calculate values that are constant for every pixel in the triangle. var v0 = p2 - p1; @@ -258,20 +261,24 @@ private static void RasterizeTriangle( var denom = d00 * d11 - d01 * d01; // If the denominator is zero, the triangle is degenerate (a line or point). - if (Math.Abs(denom) < 1e-6f) return; + if (MathF.Abs(denom) < 1e-6f) return; var baryData = new BarycentricData(v0, v1, d00, d01, d11, denom); // Calculate bounding box - var minX = (int)Math.Max(0, Math.Min(Math.Min(p1.X, p2.X), p3.X)); - var minY = (int)Math.Max(0, Math.Min(Math.Min(p1.Y, p2.Y), p3.Y)); - var maxX = (int)Math.Min(canvas.Width - 1, Math.Ceiling(Math.Max(Math.Max(p1.X, p2.X), p3.X))); - var maxY = (int)Math.Min(canvas.Height - 1, Math.Ceiling(Math.Max(Math.Max(p1.Y, p2.Y), p3.Y))); + var minX = (int)MathF.Max(0, MathF.Min(MathF.Min(p1.X, p2.X), p3.X)); + var minY = (int)MathF.Max(0, MathF.Min(MathF.Min(p1.Y, p2.Y), p3.Y)); + var maxX = (int)MathF.Min(canvas.Width - 1, MathF.Ceiling(MathF.Max(MathF.Max(p1.X, p2.X), p3.X))); + var maxY = (int)MathF.Min(canvas.Height - 1, MathF.Ceiling(MathF.Max(MathF.Max(p1.Y, p2.Y), p3.Y))); + + // Pre-calculate texture dimensions for clamping + var texWidth = textureRect.Width - 1; + var texHeight = textureRect.Height - 1; // Rasterize triangle Parallel.For((long) minY, maxY + 1, y => { - // Get a span for the current row for fast, direct memory access. + // Get a span for the current row for direct memory access. // Dangerous, but should be fine as canvas's lifetime is not at risk here. var canvasRow = canvas.DangerousGetPixelRowMemory((int) y).Span; @@ -285,20 +292,19 @@ private static void RasterizeTriangle( var texCoord = t1 * bary.X + t2 * bary.Y + t3 * bary.Z; - var texX = (int)Math.Clamp(texCoord.X * textureRect.Width, 0, textureRect.Width - 1); - var texY = (int)Math.Clamp(texCoord.Y * textureRect.Height, 0, textureRect.Height - 1); + var texX = (int)MathF.Max(0, MathF.Min(texCoord.X * textureRect.Width, texWidth)); + var texY = (int)MathF.Max(0, MathF.Min(texCoord.Y * textureRect.Height, texHeight)); var color = skin[textureRect.X + texX, textureRect.Y + texY]; - if (color.A > 10) - { - // Write directly to the canvas row's memory via the span. + if (color.A > 10) { canvasRow[x] = color; } } }); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector3 GetBarycentric(Vector2 p1, Vector2 p, in BarycentricData data) { var v2 = p - p1; @@ -335,4 +341,6 @@ private record VisibleTriangle( float Depth); private readonly record struct BarycentricData(Vector2 V0, Vector2 V1, float D00, float D01, float D11, float Denom); + + private readonly record struct PerspectiveParams(float Amount, float CameraDistance, float FocalLength); } \ No newline at end of file From d46a91dc787be164d04cd1d37669a11dc1bcbd47 Mon Sep 17 00:00:00 2001 From: ptlthg <24925519+ptlthg@users.noreply.github.com> Date: Mon, 22 Sep 2025 23:02:07 -0400 Subject: [PATCH 4/4] Load missing skull textures from NEU repo --- .../Features/Wiki/Costs/CostParserTests.cs | 11 +- .../Items/Services/ItemsIngestionService.cs | 35 +- .../Shops/Services/ShopIngestionService.cs | 2 +- .../Wiki/Services/IWikiDataService.cs | 4 +- backend/RepoAPI/Program.cs | 6 +- .../SkyblockRepo.Tests/InitializationTest.cs | 7 +- backend/SkyblockRepo/SkyblockRepo.csproj | 3 +- .../SkyblockRepo/Source/GithubRepoUpdater.cs | 190 ++++++++++ .../Source/Models/Neu/NeuItemData.cs | 192 ++++++++++ .../Source/SkyblockRepoConfiguration.cs | 67 +++- .../SkyblockRepo/Source/SkyblockRepoData.cs | 4 + .../Source/SkyblockRepoDependencyInjection.cs | 14 +- .../Source/SkyblockRepoUpdater.cs | 333 ++++++++---------- .../Source/Util/SkyblockRepoRegexUtils.cs | 28 ++ 14 files changed, 679 insertions(+), 217 deletions(-) create mode 100644 backend/SkyblockRepo/Source/GithubRepoUpdater.cs create mode 100644 backend/SkyblockRepo/Source/Models/Neu/NeuItemData.cs create mode 100644 backend/SkyblockRepo/Source/Util/SkyblockRepoRegexUtils.cs diff --git a/backend/RepoAPI.Tests/Features/Wiki/Costs/CostParserTests.cs b/backend/RepoAPI.Tests/Features/Wiki/Costs/CostParserTests.cs index a162846..b44bd28 100644 --- a/backend/RepoAPI.Tests/Features/Wiki/Costs/CostParserTests.cs +++ b/backend/RepoAPI.Tests/Features/Wiki/Costs/CostParserTests.cs @@ -1,5 +1,3 @@ -using Microsoft.Extensions.Logging; -using NSubstitute; using RepoAPI.Features.Wiki.Templates; using SkyblockRepo; using SkyblockRepo.Models; @@ -26,12 +24,15 @@ public async Task CostParser_ItemsParseCorrectly() { var input = @"{{Item/PUMPKIN_DICER|lore}}\n\n&7Cost\n&6Gold medal\n&aJacob's Ticket &8x32"; - var logger = Substitute.For>(); var config = new SkyblockRepoConfiguration { - LocalRepoPath = Path.Join(SkyblockRepoUtils.GetSolutionPath(), "..", "output") + UseNeuRepo = false, + SkyblockRepo = new RepoSettings + { + LocalPath = Path.Join(SkyblockRepoUtils.GetSolutionPath(), "..", "output") + } }; - var updater = new SkyblockRepoUpdater(config, logger); + var updater = new SkyblockRepoUpdater(config); var repo = new SkyblockRepoClient(updater); await repo.InitializeAsync(TestContext.Current.CancellationToken); diff --git a/backend/RepoAPI/Features/Items/Services/ItemsIngestionService.cs b/backend/RepoAPI/Features/Items/Services/ItemsIngestionService.cs index 946632d..c900783 100644 --- a/backend/RepoAPI/Features/Items/Services/ItemsIngestionService.cs +++ b/backend/RepoAPI/Features/Items/Services/ItemsIngestionService.cs @@ -7,6 +7,7 @@ using RepoAPI.Features.Output.Services; using RepoAPI.Features.Wiki.Services; using RepoAPI.Features.Wiki.Templates; +using SkyblockRepo; namespace RepoAPI.Features.Items.Services; @@ -19,6 +20,7 @@ public class ItemsIngestionService( JsonWriteQueue writeQueue, WikiItemsIngestionService wikiItemsIngestionService, HybridCache hybridCache, + ISkyblockRepoClient skyblockRepoClient, ILogger logger) { public async Task IngestItemsDataAsync() { @@ -46,10 +48,22 @@ public async Task IngestItemsDataAsync() { // Use a new list for items to be added to avoid modifying the context while iterating var itemsToAdd = new List(); - + foreach (var apiItem in itemsFromApi) { if (apiItem.Id is null) continue; + if (apiItem.Skin is null) { + var existingRepoItem = skyblockRepoClient.FindItem(apiItem.Id); + if (existingRepoItem?.Data?.Skin is not null) + { + apiItem.Skin = new ItemSkin() + { + Value = existingRepoItem.Data.Skin.Value, + Signature = existingRepoItem.Data.Skin.Signature + }; + } + } + if (existingItems.TryGetValue(apiItem.Id, out var existingItem)) { // Check if the item data has changed if (existingItem.Data is null || ParserUtils.DeepJsonEquals(apiItem, existingItem.Data)) { @@ -131,6 +145,25 @@ await context.SkyblockItems await WriteChangesToFile(newItem); } + foreach (var existingItem in existingItems.Values) + { + if (existingItem.Data?.Skin is not null) continue; + var repoItem = skyblockRepoClient.FindItem(existingItem.InternalId); + if (repoItem?.Data?.Skin is null) continue; + + existingItem.Data ??= new ItemResponse(); + existingItem.Data.Id ??= existingItem.InternalId; + existingItem.Data.Skin = new ItemSkin() + { + Value = repoItem.Data.Skin.Value, + Signature = repoItem.Data.Skin.Signature + }; + + context.SkyblockItems.Update(existingItem); + await WriteChangesToFile(existingItem); + updatedCount++; + } + if (itemsToAdd.Count != 0) { context.SkyblockItems.AddRange(itemsToAdd); } diff --git a/backend/RepoAPI/Features/Shops/Services/ShopIngestionService.cs b/backend/RepoAPI/Features/Shops/Services/ShopIngestionService.cs index 54c5464..41468e3 100644 --- a/backend/RepoAPI/Features/Shops/Services/ShopIngestionService.cs +++ b/backend/RepoAPI/Features/Shops/Services/ShopIngestionService.cs @@ -26,7 +26,7 @@ public async Task FetchAndLoadDataAsync(CancellationToken ct = default) const int batchSize = 50; var newEntities = 0; var updatedEntities = 0; - var allNpcIds = await wikiService.GetAllWikiShops(); + var allNpcIds = await wikiService.GetAllWikiShopsAsync(); if (allNpcIds.Count == 0) { diff --git a/backend/RepoAPI/Features/Wiki/Services/IWikiDataService.cs b/backend/RepoAPI/Features/Wiki/Services/IWikiDataService.cs index 7956519..e48ca5f 100644 --- a/backend/RepoAPI/Features/Wiki/Services/IWikiDataService.cs +++ b/backend/RepoAPI/Features/Wiki/Services/IWikiDataService.cs @@ -29,7 +29,7 @@ public interface IWikiDataService Task> GetAllWikiNpcsAsync(); Task> GetAllWikiRecipesAsync(); Task> GetAllWikiZonesAsync(); - Task> GetAllWikiShops(); + Task> GetAllWikiShopsAsync(); Task> GetAllLootTablesAsync(); Task GetPageContentAsync(string title); Task GetAttributeListAsync(); @@ -313,7 +313,7 @@ public async Task> GetAllWikiRecipesAsync() return await GetWikiCategoryAsync("DataRecipe"); } - public async Task> GetAllWikiShops() + public async Task> GetAllWikiShopsAsync() { return await GetWikiCategoryAsync("NPC_UI_Templates"); } diff --git a/backend/RepoAPI/Program.cs b/backend/RepoAPI/Program.cs index 36186be..36b877e 100644 --- a/backend/RepoAPI/Program.cs +++ b/backend/RepoAPI/Program.cs @@ -64,7 +64,11 @@ services.AddSkyblockRepo(opt => { - opt.LocalRepoPath = Path.Join(SkyblockRepoUtils.GetSolutionPath(), "..", "output"); + opt.UseNeuRepo = true; + opt.SkyblockRepo = new RepoSettings + { + LocalPath = Path.Join(SkyblockRepoUtils.GetSolutionPath(), "..", "output") + }; }); var app = builder.Build(); diff --git a/backend/SkyblockRepo.Tests/InitializationTest.cs b/backend/SkyblockRepo.Tests/InitializationTest.cs index cde0d5c..57100e3 100644 --- a/backend/SkyblockRepo.Tests/InitializationTest.cs +++ b/backend/SkyblockRepo.Tests/InitializationTest.cs @@ -12,9 +12,12 @@ public async Task InitializeFromLocalFolder() var logger = Substitute.For>(); var config = new SkyblockRepoConfiguration { - LocalRepoPath = Path.Join(SkyblockRepoUtils.GetSolutionPath(), "..", "output") + UseNeuRepo = false, + SkyblockRepo = { + LocalPath = Path.Join(SkyblockRepoUtils.GetSolutionPath(), "..", "output") + } }; - var updater = new SkyblockRepoUpdater(config, logger); + var updater = new SkyblockRepoUpdater(config); var repo = new SkyblockRepoClient(updater); await repo.InitializeAsync(); diff --git a/backend/SkyblockRepo/SkyblockRepo.csproj b/backend/SkyblockRepo/SkyblockRepo.csproj index f470e54..65a9c16 100644 --- a/backend/SkyblockRepo/SkyblockRepo.csproj +++ b/backend/SkyblockRepo/SkyblockRepo.csproj @@ -5,7 +5,7 @@ enable preview enable - 0.1.0 + 0.2.0 SkyblockRepo SkyblockRepo A .NET package for interacting with the data from https://skyblockrepo.com @@ -24,6 +24,7 @@ + diff --git a/backend/SkyblockRepo/Source/GithubRepoUpdater.cs b/backend/SkyblockRepo/Source/GithubRepoUpdater.cs new file mode 100644 index 0000000..ab27a46 --- /dev/null +++ b/backend/SkyblockRepo/Source/GithubRepoUpdater.cs @@ -0,0 +1,190 @@ +using System.IO.Compression; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace SkyblockRepo; + +public record GithubRepoOptions( + string RepoName, + string FileStoragePath, + string ApiEndpoint, + string ZipDownloadUrl, + string? LocalRepoPath = null +); + +public interface IGithubRepoUpdater +{ + string RepoPath { get; } + Task CheckForUpdatesAsync(CancellationToken cancellationToken = default); +} + +public class GithubRepoUpdater : IGithubRepoUpdater +{ + private readonly GithubRepoOptions _options; + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + private readonly string _metaFilePath; + + /// + /// Constructor for Dependency Injection users. + /// + public GithubRepoUpdater( + GithubRepoOptions options, + IHttpClientFactory httpClientFactory, + ILogger? logger = null) + { + _options = options; + _httpClient = httpClientFactory.CreateClient("RepoUpdater"); + _logger = logger ?? NullLogger.Instance; + _metaFilePath = Path.Combine(_options.FileStoragePath, $"{_options.RepoName}-meta.json"); + } + + /// + /// Constructor for non-DI users who will manage their own HttpClient instance. + /// + public GithubRepoUpdater( + GithubRepoOptions options, + HttpClient httpClient, + ILogger? logger = null) + { + _options = options; + _httpClient = httpClient; + _logger = logger ?? NullLogger.Instance; + _metaFilePath = Path.Combine(_options.FileStoragePath, $"{_options.RepoName}-meta.json"); + } + + public bool IsUsingLocalRepo => _options.LocalRepoPath is not null; + public string RepoPath => _options.LocalRepoPath ?? Path.Combine(_options.FileStoragePath, $"data-{_options.RepoName}"); + + /// + /// Checks for repository updates and downloads them if available. + /// + /// True if an update was downloaded, otherwise false. + public async Task CheckForUpdatesAsync(CancellationToken cancellationToken = default) + { + if (IsUsingLocalRepo || cancellationToken.IsCancellationRequested) + { + return false; + } + + _logger.LogInformation("[{RepoName}] Checking for updates...", _options.RepoName); + + var existingMeta = await GetLastDownloadMeta(); + + var request = new HttpRequestMessage(HttpMethod.Get, _options.ApiEndpoint); + if (existingMeta?.ETag is not null) + { + request.Headers.IfNoneMatch.Add(new System.Net.Http.Headers.EntityTagHeaderValue(existingMeta.ETag)); + } + + _httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("RepoUpdater"); + var response = await _httpClient.SendAsync(request, cancellationToken); + + if (response.StatusCode == System.Net.HttpStatusCode.NotModified || response.Headers.ETag?.Tag == existingMeta?.ETag) + { + _logger.LogInformation("[{RepoName}] No updates found.", _options.RepoName); + return false; + } + + if (!response.IsSuccessStatusCode) + { + _logger.LogError("[{RepoName}] Failed to fetch last update from {Endpoint}! Status: {StatusCode}", + _options.RepoName, _options.ApiEndpoint, response.StatusCode); + return false; + } + + var etag = response.Headers.ETag?.Tag; + if (string.IsNullOrWhiteSpace(etag)) + { + _logger.LogWarning("[{RepoName}] API endpoint did not provide an ETag. Cannot reliably check for updates.", _options.RepoName); + return false; + } + + _logger.LogInformation("[{RepoName}] New version found! Downloading...", _options.RepoName); + await DownloadRepoAsync(etag, cancellationToken); + return true; + } + + private async Task DownloadRepoAsync(string etag, CancellationToken cancellationToken) + { + var tempPath = Path.Combine(_options.FileStoragePath, $"temp-{_options.RepoName}"); + if (Directory.Exists(tempPath)) + { + Directory.Delete(tempPath, true); + } + + try + { + _httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("RepoUpdater"); + await using var stream = await _httpClient.GetStreamAsync(_options.ZipDownloadUrl, cancellationToken); + using var archive = new ZipArchive(stream); + archive.ExtractToDirectory(tempPath, true); + + var rootFolder = Directory.GetDirectories(tempPath).FirstOrDefault(); + if (rootFolder is null) + { + _logger.LogError("[{RepoName}] Invalid zip structure! No root folder found.", _options.RepoName); + return; + } + + if (Directory.Exists(RepoPath)) + { + Directory.Delete(RepoPath, true); + } + + Directory.Move(rootFolder, RepoPath); + + await SaveDownloadMeta(new DownloadMeta(DateTimeOffset.Now, etag)); + } + catch (Exception e) + { + _logger.LogError(e, "[{RepoName}] Error downloading or extracting repo.", _options.RepoName); + } + finally + { + if (Directory.Exists(tempPath)) + { + Directory.Delete(tempPath, true); + } + } + } + + private async Task GetLastDownloadMeta() + { + if (!File.Exists(_metaFilePath)) return null; + + try + { + await using var stream = File.OpenRead(_metaFilePath); + return await JsonSerializer.DeserializeAsync(stream, _jsonSerializerOptions); + } + catch (Exception e) + { + _logger.LogError(e, "[{RepoName}] Error loading download meta.", _options.RepoName); + return null; + } + } + + private async Task SaveDownloadMeta(DownloadMeta meta) + { + try + { + var tempFile = _metaFilePath + ".tmp"; + await using var stream = File.Create(tempFile); + await JsonSerializer.SerializeAsync(stream, meta, _jsonSerializerOptions); + await stream.FlushAsync(); + stream.Close(); + + File.Move(tempFile, _metaFilePath, true); + _logger.LogInformation("[{RepoName}] Saved download meta successfully!", _options.RepoName); + } + catch (Exception e) + { + _logger.LogError(e, "[{RepoName}] Error saving download meta.", _options.RepoName); + } + } + + private record DownloadMeta(DateTimeOffset LastUpdated, string ETag); +} \ No newline at end of file diff --git a/backend/SkyblockRepo/Source/Models/Neu/NeuItemData.cs b/backend/SkyblockRepo/Source/Models/Neu/NeuItemData.cs new file mode 100644 index 0000000..7ef20cb --- /dev/null +++ b/backend/SkyblockRepo/Source/Models/Neu/NeuItemData.cs @@ -0,0 +1,192 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace SkyblockRepo.Models.Neu; + +/// +/// The top-level object representing an item from the NotEnoughUpdates repository. +/// +public record NeuItemData( + [property: JsonPropertyName("itemid")] string ItemId, + [property: JsonPropertyName("displayname")] string DisplayName, + [property: JsonPropertyName("nbttag")] string NbtTag, + [property: JsonPropertyName("damage")] int Damage, + [property: JsonPropertyName("lore")] List Lore, + [property: JsonPropertyName("internalname")] string InternalName +) +{ + [property: JsonPropertyName("recipes")] public List? Recipes { get; init; } + [property: JsonPropertyName("recipe")] public NeuCraftingRecipeImplicit? Recipe { get; init; } + [property: JsonPropertyName("clickcommand")] public string? ClickCommand { get; init; } + [property: JsonPropertyName("modver")] public string? ModVer { get; init; } + [property: JsonPropertyName("useneucraft")] public bool? UseNeuCraft { get; init; } + [property: JsonPropertyName("infoType")] public string? InfoType { get; init; } + [property: JsonPropertyName("info")] public List? Info { get; init; } + [property: JsonPropertyName("crafttext")] public string? CraftText { get; init; } +} + +public class RecipeConverter : JsonConverter +{ + public override NeuRecipeBase? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + // Create a copy of the reader to peek ahead + var readerClone = reader; + + if (readerClone.TokenType != JsonTokenType.StartObject) + { + throw new JsonException("Expected the start of an object."); + } + + // Parse the JSON object into a JsonDocument to inspect it + using var jsonDoc = JsonDocument.ParseValue(ref reader); + + // Find the "type" property to determine which concrete class to use + if (!jsonDoc.RootElement.TryGetProperty("type", out var typeProperty) || typeProperty.ValueKind != JsonValueKind.String) + { + throw new JsonException("Recipe object is missing a 'type' property or it's not a string."); + } + + var recipeType = typeProperty.GetString(); + + // Deserialize the document back into the specific derived type + return recipeType switch + { + "crafting" => jsonDoc.RootElement.Deserialize(options), + "forge" => jsonDoc.RootElement.Deserialize(options), + "trade" => jsonDoc.RootElement.Deserialize(options), + "drops" => jsonDoc.RootElement.Deserialize(options), + "npc_shop" => jsonDoc.RootElement.Deserialize(options), + "katgrade" => jsonDoc.RootElement.Deserialize(options), + _ => throw new JsonException($"Unknown recipe type '{recipeType}'.") + }; + } + + public override void Write(Utf8JsonWriter writer, NeuRecipeBase value, JsonSerializerOptions options) + { + // This delegates the writing logic back to the default serializer + JsonSerializer.Serialize(writer, (object)value, options); + } +} + +public record NeuDetailedDrop( + [property: JsonPropertyName("id")] string Id +) +{ + [property: JsonPropertyName("chance")] public string? Chance { get; init; } + [property: JsonPropertyName("extra")] public List? Extra { get; init; } +} + +// The "drops" property in DropsRecipe can be either a simple string or a complex object. +// A custom JsonConverter is needed to handle this. +public class NeuDropConverter : JsonConverter +{ + public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + using var doc = JsonDocument.ParseValue(ref reader); + var root = doc.RootElement; + + if (root.ValueKind == JsonValueKind.String) + { + return root.GetString(); + } + + if (root.ValueKind == JsonValueKind.Object) + { + // Re-use the serializer to deserialize into the specific object type + return root.Deserialize(options); + } + + throw new JsonException("Expected a string or an object for a drop."); + } + + public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) + { + // Not needed for deserialization-focused classes + throw new NotImplementedException(); + } +} + +[JsonConverter(typeof(RecipeConverter))] +public record NeuRecipeBase +{ + [JsonPropertyName("type")] public required string Type { get; init; } +} + +public record NeuCraftingRecipeImplicit +{ + [JsonPropertyName("count")] public int? Count { get; init; } + [JsonPropertyName("crafttext")] public string? CraftText { get; init; } + [JsonPropertyName("overrideOutputId")] public string? OverrideOutputId { get; init; } + [JsonPropertyName("supercraftable")] public bool? Supercraftable { get; init; } + [JsonPropertyName("A1")] public string? A1 { get; init; } + [JsonPropertyName("A2")] public string? A2 { get; init; } + [JsonPropertyName("A3")] public string? A3 { get; init; } + [JsonPropertyName("B1")] public string? B1 { get; init; } + [JsonPropertyName("B2")] public string? B2 { get; init; } + [JsonPropertyName("B3")] public string? B3 { get; init; } + [JsonPropertyName("C1")] public string? C1 { get; init; } + [JsonPropertyName("C2")] public string? C2 { get; init; } + [JsonPropertyName("C3")] public string? C3 { get; init; } +} + +public record NeuCraftingRecipe : NeuRecipeBase +{ + [JsonPropertyName("count")] public int? Count { get; init; } + [JsonPropertyName("crafttext")] public string? CraftText { get; init; } + [JsonPropertyName("overrideOutputId")] public string? OverrideOutputId { get; init; } + [JsonPropertyName("supercraftable")] public bool? Supercraftable { get; init; } + [JsonPropertyName("A1")] public string? A1 { get; init; } + [JsonPropertyName("A2")] public string? A2 { get; init; } + [JsonPropertyName("A3")] public string? A3 { get; init; } + [JsonPropertyName("B1")] public string? B1 { get; init; } + [JsonPropertyName("B2")] public string? B2 { get; init; } + [JsonPropertyName("B3")] public string? B3 { get; init; } + [JsonPropertyName("C1")] public string? C1 { get; init; } + [JsonPropertyName("C2")] public string? C2 { get; init; } + [JsonPropertyName("C3")] public string? C3 { get; init; } +} + +public record NeuForgeRecipe : NeuRecipeBase +{ + [JsonPropertyName("inputs")] public required List Inputs { get; init; } + [JsonPropertyName("duration")] public required int Duration { get; init; } + [JsonPropertyName("overrideOutputId")] public string? OverrideOutputId { get; init; } + [JsonPropertyName("count")] public double? Count { get; init; } + [JsonPropertyName("hotmLevel")] public int? HotmLevel { get; init; } +} + +public record NeuTradeRecipe : NeuRecipeBase +{ + [JsonPropertyName("result")] public required string Result { get; init; } + [JsonPropertyName("cost")] public required string Cost { get; init; } + [JsonPropertyName("min")] public int? Min { get; init; } + [JsonPropertyName("max")] public int? Max { get; init; } +} + +public record NeuDropsRecipe : NeuRecipeBase +{ + [JsonPropertyName("name")] public required string Name { get; init; } + [JsonPropertyName("drops")] public required List Drops { get; init; } + [JsonPropertyName("level")] public int? Level { get; init; } + [JsonPropertyName("coins")] public int? Coins { get; init; } + [JsonPropertyName("xp")] public int? Xp { get; init; } + [JsonPropertyName("combat_xp")] public int? CombatXp { get; init; } + [JsonPropertyName("render")] public string? Render { get; init; } + [JsonPropertyName("extra")] public List? Extra { get; init; } + [JsonPropertyName("panorama")] public string? Panorama { get; init; } +} + +public record NeuNpcShopRecipe : NeuRecipeBase +{ + [JsonPropertyName("cost")] public required List Cost { get; init; } + [JsonPropertyName("result")] public required string Result { get; init; } +} + +public record NeuKatGradeRecipe : NeuRecipeBase +{ + [JsonPropertyName("input")] public required string Input { get; init; } + [JsonPropertyName("output")] public required string Output { get; init; } + [JsonPropertyName("coins")] public required int Coins { get; init; } + [JsonPropertyName("time")] public required int Time { get; init; } + [JsonPropertyName("items")] public List? Items { get; init; } +} \ No newline at end of file diff --git a/backend/SkyblockRepo/Source/SkyblockRepoConfiguration.cs b/backend/SkyblockRepo/Source/SkyblockRepoConfiguration.cs index c653af3..ad538f4 100644 --- a/backend/SkyblockRepo/Source/SkyblockRepoConfiguration.cs +++ b/backend/SkyblockRepo/Source/SkyblockRepoConfiguration.cs @@ -3,31 +3,66 @@ namespace SkyblockRepo; public class SkyblockRepoConfiguration { /// - /// The path where repo files will be stored to. Defaults to an OS-appropriate folder - /// Ex: %LOCALAPPDATA%/SkyblockRepo on Windows (adapts for other OSes) + /// The root path where all repo files will be stored. /// - public string FileStoragePath { get; set; } = + public string FileStoragePath { get; set; } = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "SkyblockRepo"); - + + /// + /// Set to true to also download and use data from the NotEnoughUpdates repository. + /// + public bool UseNeuRepo { get; set; } = false; + + /// + /// Settings for the primary SkyblockRepo. + /// + public RepoSettings SkyblockRepo { get; set; } = new() + { + Name = "skyblockrepo", + Url = "https://github.com/SkyblockRepo/Repo", + ZipPath = "/archive/refs/heads/main.zip", + ApiEndpoint = "https://api.github.com/repos/SkyblockRepo/Repo/commits/main" + }; + + /// + /// Settings for the NotEnoughUpdates repository. + /// + public RepoSettings NeuRepo { get; set; } = new() + { + Name = "neu", + Url = "https://github.com/NotEnoughUpdates/NotEnoughUpdates-REPO", + ZipPath = "/archive/refs/heads/master.zip", + ApiEndpoint = "https://api.github.com/repos/NotEnoughUpdates/NotEnoughUpdates-REPO/commits/master" + }; +} + +/// +/// Contains all the settings required to update a single repository. +/// +public class RepoSettings +{ + /// + /// The unique name for this repository (e.g., "skyblockrepo", "neu"). + /// + public string Name { get; set; } = string.Empty; + /// - /// GitHub repository URL for SkyblockRepo. Defaults to https://skyblockrepo.com/repo - /// Can be changed to use a fork or a different repository. - /// Note: The repository must follow the same structure as the original SkyblockRepo repository. + /// The base URL of the repository (e.g., https://github.com/User/Repo). /// - public string SkyblockRepoUrl { get; set; } = "https://skyblockrepo.com/repo"; - + public string Url { get; set; } = string.Empty; + /// - /// Path to the zip file of the main branch of the SkyblockRepo repository. Defaults to /archive/refs/heads/main.zip + /// The relative path to the zip archive from the base URL. /// - public string SkyblockRepoZipPath { get; set; } = "/archive/refs/heads/main.zip"; - + public string ZipPath { get; set; } = string.Empty; + /// - /// The endpoint to poll for updates with ETag support. Defaults to https://api.github.com/repos/SkyblockRepo/Repo/branches/main + /// The API endpoint to poll for updates using an ETag. /// - public string SkyblockRepoApiEndpoint { get; set; } = "https://api.github.com/repos/SkyblockRepo/Repo/branches/main"; + public string ApiEndpoint { get; set; } = string.Empty; /// - /// Set this to a local path to use a local clone of the repo instead of cloning from GitHub. + /// An optional local file path to use instead of downloading. /// - public string? LocalRepoPath { get; set; } + public string? LocalPath { get; set; } } \ No newline at end of file diff --git a/backend/SkyblockRepo/Source/SkyblockRepoData.cs b/backend/SkyblockRepo/Source/SkyblockRepoData.cs index c042888..3a5e3ae 100644 --- a/backend/SkyblockRepo/Source/SkyblockRepoData.cs +++ b/backend/SkyblockRepo/Source/SkyblockRepoData.cs @@ -1,5 +1,6 @@ using System.Collections.ObjectModel; using SkyblockRepo.Models; +using SkyblockRepo.Models.Neu; namespace SkyblockRepo; @@ -9,4 +10,7 @@ public class SkyblockRepoData public ReadOnlyDictionary Items { get; set; } = new(new Dictionary()); public ReadOnlyDictionary ItemNameSearch { get; set; } = new(new Dictionary()); public ReadOnlyDictionary Pets { get; set; } = new(new Dictionary()); + + // NEU Data + public ReadOnlyDictionary NeuItems { get; set; } = new(new Dictionary()); } \ No newline at end of file diff --git a/backend/SkyblockRepo/Source/SkyblockRepoDependencyInjection.cs b/backend/SkyblockRepo/Source/SkyblockRepoDependencyInjection.cs index c4c7ca9..533016e 100644 --- a/backend/SkyblockRepo/Source/SkyblockRepoDependencyInjection.cs +++ b/backend/SkyblockRepo/Source/SkyblockRepoDependencyInjection.cs @@ -1,6 +1,5 @@ -using System.Text.Json; -using System.Text.Json.Serialization; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace SkyblockRepo; @@ -9,7 +8,16 @@ public static class SkyblockRepoDependencyInjection public static IServiceCollection AddSkyblockRepo(this IServiceCollection services, SkyblockRepoConfiguration? options = null) { options ??= new SkyblockRepoConfiguration(); services.AddSingleton(options); - services.AddSingleton(); + + services.AddSingleton(sp => + { + var config = sp.GetRequiredService(); + var factory = sp.GetRequiredService(); + var logger = sp.GetService>(); + var repoLogger = sp.GetService>(); + return new SkyblockRepoUpdater(config, factory, logger, repoLogger); + }); + services.AddSingleton(); return services; } diff --git a/backend/SkyblockRepo/Source/SkyblockRepoUpdater.cs b/backend/SkyblockRepo/Source/SkyblockRepoUpdater.cs index 309e5b6..42e40c3 100644 --- a/backend/SkyblockRepo/Source/SkyblockRepoUpdater.cs +++ b/backend/SkyblockRepo/Source/SkyblockRepoUpdater.cs @@ -1,8 +1,9 @@ using System.Collections.ObjectModel; -using System.IO.Compression; using System.Text.Json; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using SkyblockRepo.Models; +using SkyblockRepo.Models.Neu; namespace SkyblockRepo; @@ -13,156 +14,124 @@ public interface ISkyblockRepoUpdater Task ReloadRepoAsync(CancellationToken cancellationToken = default); } -public class SkyblockRepoUpdater(SkyblockRepoConfiguration configuration, ILogger logger) : ISkyblockRepoUpdater +public class SkyblockRepoUpdater : ISkyblockRepoUpdater { public static SkyblockRepoData Data { get; set; } = new(); public static Manifest? Manifest { get => Data.Manifest; set => Data.Manifest = value; } - - private readonly JsonSerializerOptions? _jsonSerializerOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase - }; - public bool UsingLocalRepo => configuration.LocalRepoPath is not null; - public string RepoPath => configuration.LocalRepoPath ?? Path.Combine(configuration.FileStoragePath, "data-skyblockrepo"); + private readonly ILogger _logger; + private readonly IGithubRepoUpdater _skyblockRepoUpdater; + private readonly IGithubRepoUpdater? _neuRepoUpdater; // Nullable for when UseNeuRepo is false + private readonly JsonSerializerOptions _jsonSerializerOptions = new() + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Converters = { new NeuDropConverter() } + }; + + public SkyblockRepoUpdater( + SkyblockRepoConfiguration configuration, + IHttpClientFactory httpClientFactory, + ILogger? logger = null, + ILogger? repoUpdaterLogger = null) + { + _logger = logger ?? NullLogger.Instance; + _skyblockRepoUpdater = CreateUpdater(configuration.SkyblockRepo, configuration.FileStoragePath, httpClientFactory, repoUpdaterLogger); + + if (configuration.UseNeuRepo) + { + _neuRepoUpdater = CreateUpdater(configuration.NeuRepo, configuration.FileStoragePath, httpClientFactory, repoUpdaterLogger); + } + } - public async Task InitializeAsync(CancellationToken cancellationToken = default) - { - if (configuration.LocalRepoPath is not null) { - await LoadLocalRepo(); - } else { - await CheckForUpdatesAsync(cancellationToken); - } - - await ReloadRepoAsync(cancellationToken); - } - - public async Task ReloadRepoAsync(CancellationToken cancellationToken = default) - { - await LoadSkyblockItems(RepoPath); - await LoadSkyblockPets(RepoPath); - } - - public async Task CheckForUpdatesAsync(CancellationToken cancellationToken = default) - { - if (UsingLocalRepo || cancellationToken.IsCancellationRequested) return; - - logger.LogInformation("Checking for updates..."); - - var existingMeta = await GetLastDownloadMeta(); - - var httpClient = new HttpClient(); - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("SkyblockRepo"); - if (existingMeta?.ETag is not null) { - httpClient.DefaultRequestHeaders.TryAddWithoutValidation("etag", existingMeta.ETag); - } - - var response = await httpClient.GetAsync(configuration.SkyblockRepoApiEndpoint, cancellationToken); - - if (response.StatusCode == System.Net.HttpStatusCode.NotModified || - (response.Headers.ETag is not null && response.Headers.ETag.ToString() == existingMeta?.ETag)) - { - await SaveDownloadMeta(new DownloadMeta() - { - LastUpdated = DateTimeOffset.Now, - ETag = response.Headers.ETag?.ToString() ?? existingMeta?.ETag, - Version = existingMeta?.Version ?? Data.Manifest?.Version ?? 1, - }); - logger.LogInformation("No updates found!"); - return; - } - - if (!response.IsSuccessStatusCode) - { - logger.LogError("Failed to fetch last update from {Endpoint}! Status: {StatusCode}", - configuration.SkyblockRepoApiEndpoint, response.StatusCode); - return; - } - - logger.LogInformation("Updates found! Downloading new repo version..."); - await DownloadRepoAsync(response.Headers.ETag?.ToString() ?? string.Empty, cancellationToken); - } - - private async Task DownloadRepoAsync(string etag, CancellationToken cancellationToken = default) - { - var tempPath = Path.Combine(configuration.FileStoragePath, "temp-skyblockrepo"); - if (Directory.Exists(tempPath)) - { - Directory.Delete(tempPath, true); - } - - var downloadPath = configuration.SkyblockRepoZipPath.StartsWith("http", StringComparison.OrdinalIgnoreCase) - ? configuration.SkyblockRepoZipPath - : configuration.SkyblockRepoUrl + configuration.SkyblockRepoZipPath; - - var httpClient = new HttpClient(); - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("SkyblockRepo"); - - var response = await httpClient.GetAsync(downloadPath, cancellationToken); - if (!response.IsSuccessStatusCode) - { - logger.LogError("Failed to download repo zip from {DownloadPath}! Status: {StatusCode}", - downloadPath, response.StatusCode); - return; - } - - try - { - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); - using var archive = new ZipArchive(stream); - archive.ExtractToDirectory(tempPath, true); - - // The zip contains a root folder named Repo-main or similar, get that folder - var rootFolder = Directory.GetDirectories(tempPath).FirstOrDefault(); - if (rootFolder is null) - { - logger.LogError("Invalid zip structure! No root folder found."); - if (Directory.Exists(tempPath)) - { - Directory.Delete(tempPath, true); - } - return; - } - - if (Directory.Exists(RepoPath)) - { - Directory.Delete(RepoPath, true); - } - - Directory.Move(rootFolder, RepoPath); - Directory.Delete(tempPath, true); - - await SaveDownloadMeta(new DownloadMeta() - { - LastUpdated = DateTimeOffset.Now, - ETag = etag, - Version = (Manifest?.Version ?? 1) + 1, - }); - - await LoadManifest(RepoPath); - } - catch (Exception e) - { - logger.LogError(e, "Error cloning repo!"); - if (Directory.Exists(tempPath)) - { - Directory.Delete(tempPath, true); - } - } - } - - private async Task LoadLocalRepo() - { - if (configuration.LocalRepoPath is null) return; - await LoadManifest(configuration.LocalRepoPath); - } + public SkyblockRepoUpdater( + SkyblockRepoConfiguration configuration, + HttpClient? httpClient = null, + ILogger? logger = null, + ILogger? repoUpdaterLogger = null) + { + _logger = logger ?? NullLogger.Instance; + var client = httpClient ?? new HttpClient(); + _skyblockRepoUpdater = CreateUpdater(configuration.SkyblockRepo, configuration.FileStoragePath, client, repoUpdaterLogger); + + if (configuration.UseNeuRepo) + { + _neuRepoUpdater = CreateUpdater(configuration.NeuRepo, configuration.FileStoragePath, client, repoUpdaterLogger); + } + } + + private static IGithubRepoUpdater CreateUpdater(RepoSettings settings, string storagePath, IHttpClientFactory factory, ILogger? logger) + { + var options = new GithubRepoOptions( + RepoName: settings.Name, + FileStoragePath: storagePath, + ApiEndpoint: settings.ApiEndpoint, + ZipDownloadUrl: settings.Url.TrimEnd('/') + settings.ZipPath, + LocalRepoPath: settings.LocalPath + ); + return new GithubRepoUpdater(options, factory, logger); + } + + private static IGithubRepoUpdater CreateUpdater(RepoSettings settings, string storagePath, HttpClient client, ILogger? logger) + { + var options = new GithubRepoOptions( + RepoName: settings.Name, + FileStoragePath: storagePath, + ApiEndpoint: settings.ApiEndpoint, + ZipDownloadUrl: settings.Url.TrimEnd('/') + settings.ZipPath, + LocalRepoPath: settings.LocalPath + ); + return new GithubRepoUpdater(options, client, logger); + } + + public async Task InitializeAsync(CancellationToken cancellationToken = default) + { + // On startup, check all repos for updates, then load all data once. + await CheckForUpdatesAsync(cancellationToken); + await ReloadRepoAsync(cancellationToken); + } + + public async Task CheckForUpdatesAsync(CancellationToken cancellationToken = default) + { + var updateTasks = new List> { _skyblockRepoUpdater.CheckForUpdatesAsync(cancellationToken) }; + if (_neuRepoUpdater is not null) + { + updateTasks.Add(_neuRepoUpdater.CheckForUpdatesAsync(cancellationToken)); + } + + var results = await Task.WhenAll(updateTasks); + + // If any of the repos were updated, trigger a single reload of all data. + if (results.Any(wasUpdated => wasUpdated)) + { + _logger.LogInformation("One or more repositories were updated. Reloading all data..."); + await ReloadRepoAsync(cancellationToken); + } + } + + public async Task ReloadRepoAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Loading data from primary SkyblockRepo..."); + var mainRepoPath = _skyblockRepoUpdater.RepoPath; + await LoadManifest(mainRepoPath); + await LoadSkyblockItems(mainRepoPath); + await LoadSkyblockPets(mainRepoPath); + + if (_neuRepoUpdater is not null) + { + _logger.LogInformation("Loading data from NEU repo..."); + var neuRepoPath = _neuRepoUpdater.RepoPath; + + await LoadNeuItems(neuRepoPath); + } + } private async Task LoadManifest(string repoPath) { var manifestPath = Path.Combine(repoPath, "manifest.json"); if (!File.Exists(manifestPath)) { - logger.LogCritical("Invalid {RepoPath}! No manifest.json found!", repoPath); + _logger.LogCritical("Invalid {RepoPath}! No manifest.json found!", repoPath); return; } @@ -170,11 +139,11 @@ private async Task LoadManifest(string repoPath) { await using var stream = File.OpenRead(manifestPath); Manifest = await JsonSerializer.DeserializeAsync(stream, _jsonSerializerOptions); - logger.LogInformation("Manifest file loaded successfully!"); + _logger.LogInformation("Manifest file loaded successfully!"); } catch (Exception e) { - logger.LogError(e, "Error loading manifest!"); + _logger.LogError(e, "Error loading manifest!"); } } @@ -188,24 +157,52 @@ private async Task LoadSkyblockItems(string repoPath) InternalId = kvp.Value.InternalId, Name = kvp.Value.Name, }); - + Data.Items = items; Data.ItemNameSearch = new ReadOnlyDictionary(nameSearch); - + return; // The key for items needs the '-' replaced with a ':' string ItemKeySelector(string fileName) => fileName.Replace("-", ":"); } - + private async Task LoadSkyblockPets(string repoPath) { var folderPath = Path.Combine(repoPath, Manifest?.Paths.Pets ?? "pets"); Data.Pets = await LoadDataAsync(folderPath, DefaultKeySelector); } - + private static string DefaultKeySelector(string file) => file; + private async Task LoadNeuItems(string repoPath) + { + var folderPath = Path.Combine(repoPath, "items"); + var neuItems = await LoadDataAsync(folderPath, DefaultKeySelector); + + _logger.LogInformation("Loaded {Count} NEU items", neuItems.Count); + + foreach (var (id, item) in neuItems) + { + if (item.ItemId != "minecraft:skull") continue; + + var sbRepoItem = Data.Items.GetValueOrDefault(id); + if (sbRepoItem is null || sbRepoItem.Data?.Skin is not null) continue; + + var extractedSkin = SkyblockRepoRegexUtils.ExtractSkullTexture(item.NbtTag); + if (extractedSkin is null) continue; + + sbRepoItem.Data ??= new SkyblockItemResponse(); + sbRepoItem.Data.Skin = new ItemSkin() + { + Value = extractedSkin.Value, + Signature = extractedSkin.Signature + }; + } + + Data.NeuItems = neuItems; + } + /// /// Loads and deserializes JSON files from a specified folder into a read-only dictionary. /// @@ -214,13 +211,13 @@ private async Task LoadSkyblockPets(string repoPath) /// A function that takes a file name (without extension) and returns the desired dictionary key. /// A read-only dictionary of the loaded data. private async Task> LoadDataAsync( - string folderPath, - Func keySelector) where TModel : class + string folderPath, + Func keySelector) where TModel : class { var modelName = typeof(TModel).Name; if (!Directory.Exists(folderPath)) { - logger.LogWarning("Directory for {ModelName} not found: {Path}", modelName, folderPath); + _logger.LogWarning("Directory for {ModelName} not found: {Path}", modelName, folderPath); return new ReadOnlyDictionary(new Dictionary()); } @@ -244,45 +241,11 @@ await Parallel.ForEachAsync(files, } catch (Exception e) { - logger.LogError(e, "Error loading {ModelName} from {FilePath}!", modelName, filePath); + _logger.LogError(e, "Error loading {ModelName} from {FilePath}!", modelName, filePath); } }); - logger.LogInformation("Loaded {Count} {ModelName} models successfully!", data.Count, modelName); + _logger.LogInformation("Loaded {Count} {ModelName} models successfully!", data.Count, modelName); return new ReadOnlyDictionary(data); } - - private async Task GetLastDownloadMeta() - { - var metaPath = Path.Combine(configuration.FileStoragePath, "meta.json"); - if (!File.Exists(metaPath)) return null; - - try - { - await using var stream = File.OpenRead(metaPath); - return await JsonSerializer.DeserializeAsync(stream, _jsonSerializerOptions); - } catch (Exception e) - { - logger.LogError(e, "Error loading download meta!"); - return null; - } - } - - private async Task SaveDownloadMeta(DownloadMeta meta) - { - var metaPath = Path.Combine(configuration.FileStoragePath, "meta.json"); - try - { - await using var stream = File.Create(metaPath + ".tmp"); - await JsonSerializer.SerializeAsync(stream, meta, _jsonSerializerOptions); - await stream.FlushAsync(); - stream.Close(); - - File.Move(metaPath + ".tmp", metaPath, true); - logger.LogInformation("Saved download meta successfully!"); - } catch (Exception e) - { - logger.LogError(e, "Error saving download meta!"); - } - } } \ No newline at end of file diff --git a/backend/SkyblockRepo/Source/Util/SkyblockRepoRegexUtils.cs b/backend/SkyblockRepo/Source/Util/SkyblockRepoRegexUtils.cs new file mode 100644 index 0000000..2c31970 --- /dev/null +++ b/backend/SkyblockRepo/Source/Util/SkyblockRepoRegexUtils.cs @@ -0,0 +1,28 @@ +using System.Text.RegularExpressions; + +namespace SkyblockRepo; + +public static partial class SkyblockRepoRegexUtils +{ + public record SkullTextureInfo(string Value, string Signature); + + public static SkullTextureInfo? ExtractSkullTexture(string nbtTag) + { + if (string.IsNullOrWhiteSpace(nbtTag)) + { + return null; + } + + var match = SkullTextureRegex().Match(nbtTag); + + if (!match.Success) return null; + + var signature = match.Groups["signature"].Value; + var value = match.Groups["value"].Value; + return new SkullTextureInfo(value, signature); + + } + + [GeneratedRegex(@"textures:\[0:\{(?=.*?Signature:""(?.*?)"")(?=.*?Value:""(?.*?)"").*?\}", RegexOptions.Compiled)] + private static partial Regex SkullTextureRegex(); +} \ No newline at end of file