From f10e9be223ef4deed654c76891e6079264473784 Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Mon, 11 May 2020 01:31:16 -0700 Subject: [PATCH 01/19] Simple PSF insert/query work --- cs/FASTER.sln | 11 + .../FasterKVDiskReadBenchmark/Program.cs | 2 + .../FasterPSFSample/BlittableOrders.cs | 75 ++ cs/playground/FasterPSFSample/ColorKey.cs | 21 + cs/playground/FasterPSFSample/Constants.cs | 31 + cs/playground/FasterPSFSample/Context.cs | 12 + cs/playground/FasterPSFSample/FPSF.cs | 51 + .../FasterPSFSample/FasterPSFSample.cs | 130 ++ .../FasterPSFSample/FasterPSFSample.csproj | 12 + cs/playground/FasterPSFSample/IOrders.cs | 17 + cs/playground/FasterPSFSample/Input.cs | 9 + cs/playground/FasterPSFSample/Key.cs | 34 + cs/playground/FasterPSFSample/LogFiles.cs | 41 + cs/playground/FasterPSFSample/NoSerializer.cs | 17 + cs/playground/FasterPSFSample/ObjectOrders.cs | 94 ++ cs/playground/FasterPSFSample/OrdersBinKey.cs | 29 + cs/playground/FasterPSFSample/Output.cs | 15 + cs/playground/FasterPSFSample/ParseArgs.cs | 50 + cs/playground/FasterPSFSample/SizeKey.cs | 20 + cs/src/core/Allocator/AllocatorBase.cs | 4 +- .../Allocator/VarLenBlittableAllocator.cs | 4 +- cs/src/core/ClientSession/ClientSession.cs | 18 +- cs/src/core/Index/Common/Contexts.cs | 7 +- cs/src/core/Index/FASTER/FASTER.cs | 64 +- cs/src/core/Index/FASTER/FASTERBase.cs | 2 + cs/src/core/Index/FASTER/FASTERImpl.cs | 105 +- cs/src/core/Index/FASTER/FASTERThread.cs | 19 +- cs/src/core/Index/Interfaces/IFasterKV.cs | 98 ++ .../core/Index/PSF/FasterKVPSFDefinition.cs | 74 ++ cs/src/core/Index/PSF/FasterKVProviderData.cs | 44 + cs/src/core/Index/PSF/FasterPSFImpl.cs | 1063 +++++++++++++++++ .../core/Index/PSF/FasterPSFRegistration.cs | 80 ++ cs/src/core/Index/PSF/IChainPost.cs | 14 + cs/src/core/Index/PSF/IExecutePSF.cs | 37 + cs/src/core/Index/PSF/IPSF.cs | 17 + .../core/Index/PSF/IPSFCreateProviderData.cs | 16 + cs/src/core/Index/PSF/IPSFDefinition.cs | 31 + cs/src/core/Index/PSF/IQueryPSF.cs | 24 + cs/src/core/Index/PSF/PSF.cs | 41 + cs/src/core/Index/PSF/PSFCompositeKey.cs | 121 ++ cs/src/core/Index/PSF/PSFContext.cs | 12 + cs/src/core/Index/PSF/PSFFunctions.cs | 77 ++ cs/src/core/Index/PSF/PSFGroup.cs | 250 ++++ cs/src/core/Index/PSF/PSFInputSecondary.cs | 114 ++ cs/src/core/Index/PSF/PSFManager.cs | 210 ++++ cs/src/core/Index/PSF/PSFOperationStatus.cs | 15 + cs/src/core/Index/PSF/PSFOutput.cs | 94 ++ cs/src/core/Index/PSF/PSFQuerySession.cs | 285 +++++ cs/src/core/Index/PSF/PSFQuerySettings.cs | 31 + cs/src/core/Index/PSF/PSFReadArgs.cs | 17 + .../core/Index/PSF/PSFRegistrationSettings.cs | 49 + cs/src/core/Index/PSF/PSFUpdateArgs.cs | 11 + cs/src/core/Index/PSF/PSFValue.cs | 67 ++ cs/test/RecoveryTests.cs | 1 - 54 files changed, 3748 insertions(+), 39 deletions(-) create mode 100644 cs/playground/FasterPSFSample/BlittableOrders.cs create mode 100644 cs/playground/FasterPSFSample/ColorKey.cs create mode 100644 cs/playground/FasterPSFSample/Constants.cs create mode 100644 cs/playground/FasterPSFSample/Context.cs create mode 100644 cs/playground/FasterPSFSample/FPSF.cs create mode 100644 cs/playground/FasterPSFSample/FasterPSFSample.cs create mode 100644 cs/playground/FasterPSFSample/FasterPSFSample.csproj create mode 100644 cs/playground/FasterPSFSample/IOrders.cs create mode 100644 cs/playground/FasterPSFSample/Input.cs create mode 100644 cs/playground/FasterPSFSample/Key.cs create mode 100644 cs/playground/FasterPSFSample/LogFiles.cs create mode 100644 cs/playground/FasterPSFSample/NoSerializer.cs create mode 100644 cs/playground/FasterPSFSample/ObjectOrders.cs create mode 100644 cs/playground/FasterPSFSample/OrdersBinKey.cs create mode 100644 cs/playground/FasterPSFSample/Output.cs create mode 100644 cs/playground/FasterPSFSample/ParseArgs.cs create mode 100644 cs/playground/FasterPSFSample/SizeKey.cs create mode 100644 cs/src/core/Index/PSF/FasterKVPSFDefinition.cs create mode 100644 cs/src/core/Index/PSF/FasterKVProviderData.cs create mode 100644 cs/src/core/Index/PSF/FasterPSFImpl.cs create mode 100644 cs/src/core/Index/PSF/FasterPSFRegistration.cs create mode 100644 cs/src/core/Index/PSF/IChainPost.cs create mode 100644 cs/src/core/Index/PSF/IExecutePSF.cs create mode 100644 cs/src/core/Index/PSF/IPSF.cs create mode 100644 cs/src/core/Index/PSF/IPSFCreateProviderData.cs create mode 100644 cs/src/core/Index/PSF/IPSFDefinition.cs create mode 100644 cs/src/core/Index/PSF/IQueryPSF.cs create mode 100644 cs/src/core/Index/PSF/PSF.cs create mode 100644 cs/src/core/Index/PSF/PSFCompositeKey.cs create mode 100644 cs/src/core/Index/PSF/PSFContext.cs create mode 100644 cs/src/core/Index/PSF/PSFFunctions.cs create mode 100644 cs/src/core/Index/PSF/PSFGroup.cs create mode 100644 cs/src/core/Index/PSF/PSFInputSecondary.cs create mode 100644 cs/src/core/Index/PSF/PSFManager.cs create mode 100644 cs/src/core/Index/PSF/PSFOperationStatus.cs create mode 100644 cs/src/core/Index/PSF/PSFOutput.cs create mode 100644 cs/src/core/Index/PSF/PSFQuerySession.cs create mode 100644 cs/src/core/Index/PSF/PSFQuerySettings.cs create mode 100644 cs/src/core/Index/PSF/PSFReadArgs.cs create mode 100644 cs/src/core/Index/PSF/PSFRegistrationSettings.cs create mode 100644 cs/src/core/Index/PSF/PSFUpdateArgs.cs create mode 100644 cs/src/core/Index/PSF/PSFValue.cs diff --git a/cs/FASTER.sln b/cs/FASTER.sln index 088989ea2..0ed7a52ce 100644 --- a/cs/FASTER.sln +++ b/cs/FASTER.sln @@ -52,6 +52,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ClassRecoveryDurablity", "p EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FasterKVDiskReadBenchmark", "playground\FasterKVDiskReadBenchmark\FasterKVDiskReadBenchmark.csproj", "{642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FasterPSFSample", "playground\FasterPSFSample\FasterPSFSample.csproj", "{BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -196,6 +198,14 @@ Global {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Release|Any CPU.Build.0 = Release|x64 {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Release|x64.ActiveCfg = Release|x64 {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Release|x64.Build.0 = Release|x64 + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Debug|x64.ActiveCfg = Debug|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Debug|x64.Build.0 = Debug|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Release|Any CPU.Build.0 = Release|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Release|x64.ActiveCfg = Release|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -220,6 +230,7 @@ Global {859F76F4-93D8-4D60-BF9A-363E217FA247} = {E6026D6A-01C5-4582-B2C1-64751490DABE} {95AC8766-84F9-4E95-B2E9-2169B6375FB2} = {E6026D6A-01C5-4582-B2C1-64751490DABE} {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49} = {E6026D6A-01C5-4582-B2C1-64751490DABE} + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386} = {E6026D6A-01C5-4582-B2C1-64751490DABE} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A0750637-2CCB-4139-B25E-F2CE740DCFAC} diff --git a/cs/playground/FasterKVDiskReadBenchmark/Program.cs b/cs/playground/FasterKVDiskReadBenchmark/Program.cs index 5870ccd92..1c41adca2 100644 --- a/cs/playground/FasterKVDiskReadBenchmark/Program.cs +++ b/cs/playground/FasterKVDiskReadBenchmark/Program.cs @@ -7,6 +7,8 @@ using System.Threading.Tasks; using FASTER.core; +#pragma warning disable CS0162 // Unreachable code detected + namespace FasterKVDiskReadBenchmark { public class Program diff --git a/cs/playground/FasterPSFSample/BlittableOrders.cs b/cs/playground/FasterPSFSample/BlittableOrders.cs new file mode 100644 index 000000000..46994b969 --- /dev/null +++ b/cs/playground/FasterPSFSample/BlittableOrders.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System; +using System.Drawing; + +namespace FasterPSFSample +{ + public struct BlittableOrders : IOrders + { + // Colors, strings, and enums are not blittable so we use int + public int Size { get; set; } + + public int Color { get; set; } + + public int NumSold { get; set; } + + public BlittableOrders(Constants.Size size, Color color, int numSold) + { + this.Size = (int)size; + this.Color = color.ToArgb(); + this.NumSold = numSold; + } + + public (int, int, int) MemberTuple => (this.Size, this.Color, this.NumSold); + + public override string ToString() => $"{(Constants.Size)this.Size}, {Constants.ColorDict[this.Color].Name}, {NumSold}"; + + public class Functions : IFunctions, Context> + { + public void ConcurrentReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) + => dst.Value = value; + + public bool ConcurrentWriter(ref Key key, ref BlittableOrders src, ref BlittableOrders dst) + { + dst = src; + return true; + } + + public void CopyUpdater(ref Key key, ref Input input, ref BlittableOrders oldValue, ref BlittableOrders newValue) + => throw new NotImplementedException(); + + public void InitialUpdater(ref Key key, ref Input input, ref BlittableOrders value) + => throw new NotImplementedException(); + + public bool InPlaceUpdater(ref Key key, ref Input input, ref BlittableOrders value) + => throw new NotImplementedException(); + + public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) + => throw new NotImplementedException(); + + public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) + { + if (output.Value.MemberTuple != key.MemberTuple) + throw new Exception("Read mismatch error!"); + } + + public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) + => throw new NotImplementedException(); + + public void SingleReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) + => dst.Value = value; + + public void SingleWriter(ref Key key, ref BlittableOrders src, ref BlittableOrders dst) + => dst = src; + + public void UpsertCompletionCallback(ref Key key, ref BlittableOrders value, Context context) + => throw new NotImplementedException(); + + public void DeleteCompletionCallback(ref Key key, Context context) + => throw new NotImplementedException(); + } + } +} diff --git a/cs/playground/FasterPSFSample/ColorKey.cs b/cs/playground/FasterPSFSample/ColorKey.cs new file mode 100644 index 000000000..e8859ec7a --- /dev/null +++ b/cs/playground/FasterPSFSample/ColorKey.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System.Drawing; + +namespace FasterPSFSample +{ + public struct ColorKey : IFasterEqualityComparer + { + public int Color; + + public ColorKey(Color color) => this.Color = color.ToArgb(); + + public override string ToString() => Constants.ColorDict[this.Color].Name; + + public long GetHashCode64(ref ColorKey key) => Utility.GetHashCode(key.Color); + + public bool Equals(ref ColorKey k1, ref ColorKey k2) => k1.Color == k2.Color; + } +} diff --git a/cs/playground/FasterPSFSample/Constants.cs b/cs/playground/FasterPSFSample/Constants.cs new file mode 100644 index 000000000..d154abd98 --- /dev/null +++ b/cs/playground/FasterPSFSample/Constants.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Drawing; + +namespace FasterPSFSample +{ + public static class Constants + { + // Colors, strings, and enums are not blittable so we store int + public enum Size + { + Small, + Medium, + Large, + XLarge, + NumSizes + } + + static internal Dictionary ColorDict = new Dictionary + { + [Color.Black.ToArgb()] = Color.Black, + [Color.Red.ToArgb()] = Color.Red, + [Color.Green.ToArgb()] = Color.Green, + [Color.Blue.ToArgb()] = Color.Blue + }; + + static internal Color[] Colors = { Color.Black, Color.Red, Color.Green, Color.Blue }; + } +} diff --git a/cs/playground/FasterPSFSample/Context.cs b/cs/playground/FasterPSFSample/Context.cs new file mode 100644 index 000000000..a0e72f679 --- /dev/null +++ b/cs/playground/FasterPSFSample/Context.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Collections.Generic; + +namespace FasterPSFSample +{ + public class Context + { + public List Value { get; set; } = new List(); + } +} diff --git a/cs/playground/FasterPSFSample/FPSF.cs b/cs/playground/FasterPSFSample/FPSF.cs new file mode 100644 index 000000000..caa0ab2cb --- /dev/null +++ b/cs/playground/FasterPSFSample/FPSF.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FasterPSFSample +{ + class FPSF + where TValue : IOrders, new() + where TOutput : new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + internal IFasterKV, TFunctions> fht { get; set; } + + private LogFiles logFiles; + + internal PSF SizePsf; + internal PSF ColorPsf; + + internal FPSF(bool useObjectValues, bool useReadCache) + { + this.logFiles = new LogFiles(useObjectValues, useReadCache); + + this.fht = new FasterKV, TFunctions>( + 1L << 20, new TFunctions(), this.logFiles.LogSettings, + null, // TODO: add checkpoints + useObjectValues + ? new SerializerSettings { valueSerializer = () => new TSerializer() } : null); + + this.SizePsf = fht.RegisterPSF("sizePsf", (k, v) => new SizeKey((Constants.Size)v.Size)); + this.ColorPsf = fht.RegisterPSF("colorPsf", (k, v) => new ColorKey(Constants.ColorDict[v.Color])); + } + + internal void Close() + { + if (!(this.fht is null)) + { + this.fht.Dispose(); + this.fht = null; + } + if (!(this.logFiles is null)) + { + this.logFiles.Close(); + this.logFiles = null; + } + } + } +} diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs new file mode 100644 index 000000000..d220162ad --- /dev/null +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System; +using System.Diagnostics; + +namespace FasterPSFSample +{ + public partial class FasterPSFSample + { + private const int UpsertCount = 100; + + static void Main(string[] argv) + { + if (!ParseArgs(argv)) + return; + + if (useObjectValue) // TODO add VarLenValue + RunSample, ObjectOrders.Functions, ObjectOrders.Serializer>(); + else + RunSample, BlittableOrders.Functions, NoSerializer>(); + return; + } + + internal static void RunSample() + where TValue : IOrders, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + var fpsf = new FPSF(useObjectValue, useReadCache: true); + try + { + RunUpserts(fpsf); + RunReads(fpsf); + RunPSFs(fpsf); + } + finally + { + fpsf.Close(); + } + + Console.WriteLine("Press to end"); + Console.ReadLine(); + } + + internal static void RunUpserts(FPSF fpsf) + where TValue : IOrders, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine("Writing keys from 0 to {0} to FASTER", UpsertCount); + + var rng = new Random(13); + using (var session = fpsf.fht.NewSession()) + { + var context = new Context(); + + for (int i = 0; i < UpsertCount; i++) + { + var key = new Key((Constants.Size)rng.Next((int)Constants.Size.NumSizes), + Constants.Colors[rng.Next((int)Constants.Colors.Length)], + rng.Next(OrdersBinKey.MaxOrders)); + var value = new TValue { Size = key.Size, Color = key.Color, NumSold = key.NumSold }; + session.Upsert(ref key, ref value, context, 0); + } + } + + Console.WriteLine($"Upserted {UpsertCount} elements"); + } + + internal static void RunReads(FPSF fpsf) + where TValue : IOrders, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine("Reading {0} random keys from FASTER", UpsertCount); + + var rng = new Random(0); + int statusPending = 0; + var output = new TOutput(); + var input = default(Input); + var context = new Context(); + var readCount = UpsertCount * 2; + + using (var session = fpsf.fht.NewSession()) + { + for (int i = 0; i < UpsertCount; i++) + { + var key = new Key((Constants.Size)rng.Next((int)Constants.Size.NumSizes), + Constants.Colors[rng.Next((int)Constants.Colors.Length)], + rng.Next(OrdersBinKey.MaxOrders)); + var status = session.Read(ref key, ref input, ref output, context, 0); + + if (status == Status.OK && output.Value.MemberTuple != key.MemberTuple) + throw new Exception($"Error: Value does not match key in {nameof(RunReads)}"); + } + + session.CompletePending(true); + } + Console.WriteLine($"Read {readCount} elements with {++statusPending} Pending"); + } + + const string indent = " "; + + internal static void RunPSFs(FPSF fpsf) + where TValue : IOrders, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine("Querying PSFs from FASTER", UpsertCount); + + using (var session = fpsf.fht.NewSession()) + { + Console.Write("No join op; all Mediums: "); + foreach (var providerData in session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium))) + { + Console.WriteLine(); + ref TValue value = ref providerData.GetValue(); + Console.Write(indent + value); + } + Console.WriteLine(); + } + } + } +} diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.csproj b/cs/playground/FasterPSFSample/FasterPSFSample.csproj new file mode 100644 index 000000000..89a1694f6 --- /dev/null +++ b/cs/playground/FasterPSFSample/FasterPSFSample.csproj @@ -0,0 +1,12 @@ + + + + Exe + netcoreapp3.1 + + + + + + + diff --git a/cs/playground/FasterPSFSample/IOrders.cs b/cs/playground/FasterPSFSample/IOrders.cs new file mode 100644 index 000000000..364a722c3 --- /dev/null +++ b/cs/playground/FasterPSFSample/IOrders.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FasterPSFSample +{ + public interface IOrders + { + // Colors, strings, and enums are not blittable so we use int + int Size { get; set; } + + int Color { get; set; } + + int NumSold { get; set; } + + (int, int, int) MemberTuple => (this.Size, this.Color, this.NumSold); + } +} diff --git a/cs/playground/FasterPSFSample/Input.cs b/cs/playground/FasterPSFSample/Input.cs new file mode 100644 index 000000000..65fb779a5 --- /dev/null +++ b/cs/playground/FasterPSFSample/Input.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FasterPSFSample +{ + public struct Input + { + } +} diff --git a/cs/playground/FasterPSFSample/Key.cs b/cs/playground/FasterPSFSample/Key.cs new file mode 100644 index 000000000..3f7e0c772 --- /dev/null +++ b/cs/playground/FasterPSFSample/Key.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System.Drawing; + +namespace FasterPSFSample +{ + public struct Key : IFasterEqualityComparer, IOrders + { + public int Size { get; set; } + + public int Color { get; set; } + + public int NumSold { get; set; } + + public Key(Constants.Size size, Color color, int numSold) + { + this.Size = (int)size; + this.Color = color.ToArgb(); + this.NumSold = numSold; + } + + public (int, int, int) MemberTuple => (this.Size, this.Color, this.NumSold); + + public override string ToString() => $"{(Constants.Size)this.Size}, {Constants.ColorDict[this.Color].Name}, {NumSold}"; + + private long AsLong => (this.Size + this.Color) << 30 | this.NumSold; + + public long GetHashCode64(ref Key key) => Utility.GetHashCode(key.AsLong); + + public bool Equals(ref Key k1, ref Key k2) => k1.MemberTuple == k2.MemberTuple; + } +} diff --git a/cs/playground/FasterPSFSample/LogFiles.cs b/cs/playground/FasterPSFSample/LogFiles.cs new file mode 100644 index 000000000..1fc84a855 --- /dev/null +++ b/cs/playground/FasterPSFSample/LogFiles.cs @@ -0,0 +1,41 @@ +using FASTER.core; +using System.IO; + +namespace FasterPSFSample +{ + class LogFiles + { + private IDevice log; + private IDevice objLog; + + internal LogSettings LogSettings { get; } + + internal LogFiles(bool useObjectValue, bool useReadCache) + { + // Create files for storing data. We only use one write thread to avoid disk contention. + // We set deleteOnClose to true, so logs will auto-delete on completion. + this.log = Devices.CreateLogDevice(Path.GetTempPath() + "hlog.log", deleteOnClose: true); + if (useObjectValue) + this.objLog = Devices.CreateLogDevice(Path.GetTempPath() + "hlog.obj.log", deleteOnClose: true); + + // Define settings for log + this.LogSettings = new LogSettings { LogDevice = log, ObjectLogDevice = objLog }; + if (useReadCache) + this.LogSettings.ReadCacheSettings = new ReadCacheSettings(); + } + + internal void Close() + { + if (!(this.log is null)) + { + this.log.Close(); + this.log = null; + } + if (!(this.objLog is null)) + { + this.objLog.Close(); + this.objLog = null; + } + } + } +} diff --git a/cs/playground/FasterPSFSample/NoSerializer.cs b/cs/playground/FasterPSFSample/NoSerializer.cs new file mode 100644 index 000000000..508fa57f3 --- /dev/null +++ b/cs/playground/FasterPSFSample/NoSerializer.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System; + +namespace FasterPSFSample +{ + public class NoSerializer : BinaryObjectSerializer + { + public override void Deserialize(ref BlittableOrders obj) + => throw new NotImplementedException("NoSerializer should not be instantiated"); + + public override void Serialize(ref BlittableOrders obj) + => throw new NotImplementedException("NoSerializer should not be instantiated"); + } +} diff --git a/cs/playground/FasterPSFSample/ObjectOrders.cs b/cs/playground/FasterPSFSample/ObjectOrders.cs new file mode 100644 index 000000000..8b1352b36 --- /dev/null +++ b/cs/playground/FasterPSFSample/ObjectOrders.cs @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System; +using System.Drawing; + +namespace FasterPSFSample +{ + public class ObjectOrders : IOrders + { + public int Size { get => values[0]; set => values[0] = value; } + + public int Color { get => values[1]; set => values[1] = value; } + + public int NumSold { get => values[2]; set => values[2] = value; } + + public int[] values; + + public ObjectOrders() => throw new InvalidOperationException("Must use ctor overload"); + + public ObjectOrders(Constants.Size size, Color color, int numSold) + { + this.Size = (int)size; + this.Color = color.ToArgb(); + this.NumSold = numSold; + } + + public (int, int, int) MemberTuple => (this.Size, this.Color, this.NumSold); + + public override string ToString() => $"{(Constants.Size)this.Size}, {Constants.ColorDict[this.Color].Name}, {NumSold}"; + + public class Serializer : BinaryObjectSerializer + { + public override void Deserialize(ref ObjectOrders obj) + { + obj.values = new int[3]; + for (var ii = 0; ii < obj.values.Length; ++ii) + obj.values[ii] = reader.ReadInt32(); + } + + public override void Serialize(ref ObjectOrders obj) + { + for (var ii = 0; ii < obj.values.Length; ++ii) + writer.Write(obj.values[ii]); + } + } + + public class Functions : IFunctions, Context> + { + public void ConcurrentReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) + => dst.Value = value; + + public bool ConcurrentWriter(ref Key key, ref ObjectOrders src, ref ObjectOrders dst) + { + dst = src; + return true; + } + + public void CopyUpdater(ref Key key, ref Input input, ref ObjectOrders oldValue, ref ObjectOrders newValue) + => throw new NotImplementedException(); + + public void InitialUpdater(ref Key key, ref Input input, ref ObjectOrders value) + => throw new NotImplementedException(); + + public bool InPlaceUpdater(ref Key key, ref Input input, ref ObjectOrders value) + => throw new NotImplementedException(); + + public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) + => throw new NotImplementedException(); + + public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) + { + if (output.Value.MemberTuple != key.MemberTuple) + throw new Exception("Read mismatch error!"); + } + + public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) + => throw new NotImplementedException(); + + public void SingleReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) + => dst.Value = value; + + public void SingleWriter(ref Key key, ref ObjectOrders src, ref ObjectOrders dst) + => dst = src; + + public void UpsertCompletionCallback(ref Key key, ref ObjectOrders value, Context context) + => throw new NotImplementedException(); + + public void DeleteCompletionCallback(ref Key key, Context context) + => throw new NotImplementedException(); + } + } +} diff --git a/cs/playground/FasterPSFSample/OrdersBinKey.cs b/cs/playground/FasterPSFSample/OrdersBinKey.cs new file mode 100644 index 000000000..87db1b150 --- /dev/null +++ b/cs/playground/FasterPSFSample/OrdersBinKey.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; + +namespace FasterPSFSample +{ + public struct OrdersBinKey : IFasterEqualityComparer + { + internal const int BinSize = 100; + internal const int MaxOrders = BinSize * 10; + internal const int MaxBin = 8; // 0-based; skip the last one + + public int Bin; + + public OrdersBinKey(int bin) => this.Bin = bin; + + internal bool GetBin(int numOrders, out int bin) + { + bin = numOrders / BinSize; + return bin < MaxBin; + } + + // Make the hashcode for this distinct from size enum values + public long GetHashCode64(ref OrdersBinKey key) => Utility.GetHashCode(this.Bin + 1000); + + public bool Equals(ref OrdersBinKey k1, ref OrdersBinKey k2) => k1.Bin == k2.Bin; + } +} diff --git a/cs/playground/FasterPSFSample/Output.cs b/cs/playground/FasterPSFSample/Output.cs new file mode 100644 index 000000000..3690c0cde --- /dev/null +++ b/cs/playground/FasterPSFSample/Output.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FasterPSFSample +{ + public interface IOutput + { + public TValue Value { get; set; } + } + + public struct Output : IOutput + { + public TValue Value { get; set; } + } +} diff --git a/cs/playground/FasterPSFSample/ParseArgs.cs b/cs/playground/FasterPSFSample/ParseArgs.cs new file mode 100644 index 000000000..7b7550015 --- /dev/null +++ b/cs/playground/FasterPSFSample/ParseArgs.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; + +namespace FasterPSFSample +{ + public partial class FasterPSFSample + { + private static bool useObjectValue; + + const string ObjValuesArg = "--objValues"; + + static bool ParseArgs(string[] argv) + { + static bool Usage(string message = null) + { + Console.WriteLine(); + Console.WriteLine($"Usage: Run one or more Predicate Subset Functions (PSFs), specifying whether to use object or blittable (primitive) values."); + Console.WriteLine(); + Console.WriteLine($" {ObjValuesArg}: Use objects instead of blittable Value; default is {useObjectValue}"); + Console.WriteLine(); + if (!string.IsNullOrEmpty(message)) + { + Console.WriteLine("====== Invalid Argument(s) ======"); + Console.WriteLine(message); + Console.WriteLine(); + } + Console.WriteLine(); + return false; + } + + for (var ii = 0; ii < argv.Length; ++ii) + { + var arg = argv[ii]; + if (string.Compare(arg, ObjValuesArg, ignoreCase: true) == 0) + { + useObjectValue = true; + continue; + } + if (string.Compare(arg, "--help", ignoreCase: true) == 0 || arg == "/?" || arg == "-?") + { + return Usage(); + } + return Usage($"Unknown argument: {arg}"); + } + return true; + } + } +} diff --git a/cs/playground/FasterPSFSample/SizeKey.cs b/cs/playground/FasterPSFSample/SizeKey.cs new file mode 100644 index 000000000..484b3934c --- /dev/null +++ b/cs/playground/FasterPSFSample/SizeKey.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; + +namespace FasterPSFSample +{ + public struct SizeKey : IFasterEqualityComparer + { + public int Size; + + public SizeKey(Constants.Size size) => this.Size = (int)size; + + public override string ToString() => ((Constants.Size)this.Size).ToString(); + + public long GetHashCode64(ref SizeKey key) => Utility.GetHashCode(key.Size); + + public bool Equals(ref SizeKey k1, ref SizeKey k2) => k1.Size == k2.Size; + } +} diff --git a/cs/src/core/Allocator/AllocatorBase.cs b/cs/src/core/Allocator/AllocatorBase.cs index f6b0a130a..571e2f9ef 100644 --- a/cs/src/core/Allocator/AllocatorBase.cs +++ b/cs/src/core/Allocator/AllocatorBase.cs @@ -197,7 +197,7 @@ public unsafe abstract partial class AllocatorBase : IDisposable /// /// Buffer pool /// - protected SectorAlignedBufferPool bufferPool; + internal SectorAlignedBufferPool bufferPool; /// /// Read cache @@ -515,7 +515,7 @@ public AllocatorBase(LogSettings settings, IFasterEqualityComparer comparer // Segment size LogSegmentSizeBits = settings.SegmentSizeBits; - SegmentSize = 1 << LogSegmentSizeBits; + SegmentSize = 1L << LogSegmentSizeBits; SegmentBufferSize = 1 + (LogTotalSizeBytes / SegmentSize < 1 ? 1 : (int)(LogTotalSizeBytes / SegmentSize)); if (SegmentSize < PageSize) diff --git a/cs/src/core/Allocator/VarLenBlittableAllocator.cs b/cs/src/core/Allocator/VarLenBlittableAllocator.cs index c180d3717..e26153e60 100644 --- a/cs/src/core/Allocator/VarLenBlittableAllocator.cs +++ b/cs/src/core/Allocator/VarLenBlittableAllocator.cs @@ -154,7 +154,7 @@ public override void ShallowCopy(ref Key src, ref Key dst) Unsafe.AsPointer(ref src), Unsafe.AsPointer(ref dst), KeyLength.GetLength(ref src), - KeyLength.GetLength(ref src)); + KeyLength.GetLength(ref src)); // Note: dst may not be initialized (caller ensures space is available) } public override void ShallowCopy(ref Value src, ref Value dst) @@ -163,7 +163,7 @@ public override void ShallowCopy(ref Value src, ref Value dst) Unsafe.AsPointer(ref src), Unsafe.AsPointer(ref dst), ValueLength.GetLength(ref src), - ValueLength.GetLength(ref src)); + ValueLength.GetLength(ref src)); // Note: dst may not be initialized (caller ensures space is available) } /// diff --git a/cs/src/core/ClientSession/ClientSession.cs b/cs/src/core/ClientSession/ClientSession.cs index f15f90a6c..9069550ec 100644 --- a/cs/src/core/ClientSession/ClientSession.cs +++ b/cs/src/core/ClientSession/ClientSession.cs @@ -20,7 +20,9 @@ namespace FASTER.core /// /// /// - public sealed class ClientSession : IDisposable + public sealed partial class ClientSession : + IPSFCreateProviderData>, + IDisposable where Key : new() where Value : new() where Functions : IFunctions @@ -49,12 +51,26 @@ internal ClientSession( /// public string ID { get { return ctx.guid; } } + /// + public FasterKVProviderData Create(long logicalAddress) + { + // Looks up logicalAddress in the primary FasterKV + var psfArgs = new PSFReadArgs(new PSFInputPrimaryReadAddress(logicalAddress), + new PSFOutputPrimaryReadAddress(this.fht.hlog)); + + // Call this directly here, because we are + var status = fht.ContextPsfReadAddress(ref psfArgs, 1 /*TODO lsn*/, ctx); + var primaryOutput = psfArgs.Output as IPSFPrimaryOutput>; + return status == Status.OK ? primaryOutput.ProviderData : null; // TODO check other states + } + /// /// Dispose session /// public void Dispose() { CompletePending(true); + // TODO: this was removed; if intended, remove the method too: fht.DisposeClientSession(ID); // Session runs on a single thread if (!SupportAsync) diff --git a/cs/src/core/Index/Common/Contexts.cs b/cs/src/core/Index/Common/Contexts.cs index fbb78f668..e83c36864 100644 --- a/cs/src/core/Index/Common/Contexts.cs +++ b/cs/src/core/Index/Common/Contexts.cs @@ -17,7 +17,10 @@ internal enum OperationType RMW, UPSERT, INSERT, - DELETE + DELETE, + PSF_READ_KEY, + PSF_READ_ADDRESS, + PSF_INSERT } internal enum OperationStatus @@ -85,6 +88,8 @@ internal struct PendingContext internal long serialNum; internal HashBucketEntry entry; internal LatchOperation heldLatch; + internal PSFReadArgs psfReadArgs; + internal PSFUpdateArgs psfUpdateArgs; public void Dispose() { diff --git a/cs/src/core/Index/FASTER/FASTER.cs b/cs/src/core/Index/FASTER/FASTER.cs index 4f7990e02..a29220842 100644 --- a/cs/src/core/Index/FASTER/FASTER.cs +++ b/cs/src/core/Index/FASTER/FASTER.cs @@ -183,6 +183,14 @@ public FasterKV(long size, Functions functions, LogSettings logSettings, _systemState = default; _systemState.phase = Phase.REST; _systemState.version = 1; + + // To pass along to PSF-implementing secondary FasterKV + this.hashTableSize = size; + this.logSettings = logSettings; + this.checkpointSettings = checkpointSettings; + this.serializerSettings = serializerSettings; + this.variableLengthStructSettings = variableLengthStructSettings; + this.InitializePSFs(); } /// @@ -315,16 +323,20 @@ internal Status ContextUpsert(ref Key key, ref Value value, Context context, lon FasterExecutionContext sessionCtx) { var pcontext = default(PendingContext); - var internalStatus = InternalUpsert(ref key, ref value, ref context, ref pcontext, sessionCtx, serialNo); + var updateArgs = new PSFUpdateArgs(); + var internalStatus = InternalUpsert(ref key, ref value, ref context, ref pcontext, sessionCtx, + serialNo, ref updateArgs); Status status; - - if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) + if (internalStatus == OperationStatus.SUCCESS && this.PSFManager.HasPSFs) { - status = (Status) internalStatus; + status = this.PSFManager.Upsert(new FasterKVProviderData(this.hlog, ref key, ref value), + updateArgs.logicalAddress, updateArgs.isInserted); } else { - status = HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); } sessionCtx.serialNum = serialNo; @@ -370,6 +382,48 @@ internal Status ContextDelete(ref Key key, Context context, long serialNo, Faste return status; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfInsert(ref Key key, ref Value value, ref Input input, long serialNo, + FasterExecutionContext sessionCtx) + { + var pcontext = default(PendingContext); + var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, + ref pcontext, sessionCtx, serialNo); + var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfReadKey(ref Key key, ref PSFReadArgs psfArgs, long serialNo, + FasterExecutionContext sessionCtx) + { + var pcontext = default(PendingContext); + var internalStatus = this.PsfInternalReadKey(ref key, ref psfArgs, ref pcontext, sessionCtx, serialNo); + var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfReadAddress(ref PSFReadArgs psfArgs, long serialNo, + FasterExecutionContext sessionCtx) + { + var pcontext = default(PendingContext); + var internalStatus = this.PsfInternalReadAddress(ref psfArgs, ref pcontext, sessionCtx, serialNo); + var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } /// /// Grow the hash index diff --git a/cs/src/core/Index/FASTER/FASTERBase.cs b/cs/src/core/Index/FASTER/FASTERBase.cs index debef6709..9090eb47c 100644 --- a/cs/src/core/Index/FASTER/FASTERBase.cs +++ b/cs/src/core/Index/FASTER/FASTERBase.cs @@ -74,6 +74,8 @@ internal static class Constants public const long kInvalidAddress = 0; public const long kTempInvalidAddress = 1; public const int kFirstValidAddress = 64; + + public const int kInvalidPsfOrdinal = -1; } [StructLayout(LayoutKind.Explicit, Size = Constants.kEntriesPerBucket * 8)] diff --git a/cs/src/core/Index/FASTER/FASTERImpl.cs b/cs/src/core/Index/FASTER/FASTERImpl.cs index 851723872..55f730c6b 100644 --- a/cs/src/core/Index/FASTER/FASTERImpl.cs +++ b/cs/src/core/Index/FASTER/FASTERImpl.cs @@ -230,6 +230,7 @@ internal OperationStatus InternalRead( /// Pending context used internally to store the context of the operation. /// Session context /// Operation serial number + /// For PSFs, returns the inserted or updated LogicalAddress /// /// /// @@ -254,7 +255,8 @@ internal OperationStatus InternalRead( internal OperationStatus InternalUpsert( ref Key key, ref Value value, ref Context userContext, - ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) + ref PendingContext pendingContext, FasterExecutionContext sessionCtx, + long lsn, ref PSFUpdateArgs psfArgs) { var status = default(OperationStatus); var bucket = default(HashBucket*); @@ -263,6 +265,8 @@ internal OperationStatus InternalUpsert( var physicalAddress = default(long); var latchOperation = default(LatchOperation); var latestRecordVersion = -1; + psfArgs.logicalAddress = Constants.kInvalidAddress; + psfArgs.isInserted = false; var hash = comparer.GetHashCode64(ref key); var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); @@ -296,6 +300,8 @@ internal OperationStatus InternalUpsert( } #endregion + psfArgs.logicalAddress = logicalAddress; + // Optimization for most common case if (sessionCtx.phase == Phase.REST && logicalAddress >= hlog.ReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) { @@ -403,11 +409,13 @@ internal OperationStatus InternalUpsert( var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); RecordInfo.WriteInfo(ref hlog.GetInfo(newPhysicalAddress), sessionCtx.version, - true, false, false, + final:true, tombstone:false, invalidBit:false, latestLogicalAddress); hlog.ShallowCopy(ref key, ref hlog.GetKey(newPhysicalAddress)); functions.SingleWriter(ref key, ref value, ref hlog.GetValue(newPhysicalAddress)); + psfArgs.logicalAddress = newLogicalAddress; + psfArgs.isInserted = true; var updatedEntry = default(HashBucketEntry); updatedEntry.Tag = tag; @@ -467,7 +475,7 @@ internal OperationStatus InternalUpsert( if (status == OperationStatus.RETRY_NOW) { - return InternalUpsert(ref key, ref value, ref userContext, ref pendingContext, sessionCtx, lsn); + return InternalUpsert(ref key, ref value, ref userContext, ref pendingContext, sessionCtx, lsn, ref psfArgs); } else { @@ -1209,8 +1217,25 @@ internal OperationStatus InternalContinuePendingRead( if (hlog.GetInfoFromBytePointer(request.record.GetValidPointer()).Tombstone) return OperationStatus.NOTFOUND; - functions.SingleReader(ref pendingContext.key.Get(), ref pendingContext.input, - ref hlog.GetContextRecordValue(ref request), ref pendingContext.output); + if (pendingContext.type == OperationType.READ) + { + functions.SingleReader(ref pendingContext.key.Get(), ref pendingContext.input, + ref hlog.GetContextRecordValue(ref request), ref pendingContext.output); + } + else if (pendingContext.type == OperationType.PSF_READ_KEY) + { + pendingContext.psfReadArgs.Output.Visit(pendingContext.psfReadArgs.Input.PsfOrdinal, + ref pendingContext.key.Get(), + ref hlog.GetContextRecordValue(ref request), + isConcurrent: false); + } + else if (pendingContext.type == OperationType.PSF_READ_ADDRESS) + { + var key = default(Key); // Reading by address does not have a key + pendingContext.psfReadArgs.Output.Visit(pendingContext.psfReadArgs.Input.PsfOrdinal, + ref key, ref hlog.GetContextRecordValue(ref request), + isConcurrent: false); + } if (CopyReadsToTail || UseReadCache) { @@ -1511,10 +1536,26 @@ internal Status HandleOperationStatus( ref pendingContext.userContext, ref pendingContext, currentCtx, pendingContext.serialNum); break; + case OperationType.PSF_READ_KEY: + internalStatus = PsfInternalReadKey(ref pendingContext.key.Get(), + ref pendingContext.psfReadArgs, + ref pendingContext, currentCtx, pendingContext.serialNum); + break; + case OperationType.PSF_READ_ADDRESS: + internalStatus = PsfInternalReadAddress(ref pendingContext.psfReadArgs, + ref pendingContext, currentCtx, pendingContext.serialNum); + break; case OperationType.UPSERT: internalStatus = InternalUpsert(ref pendingContext.key.Get(), ref pendingContext.value.Get(), ref pendingContext.userContext, + ref pendingContext, currentCtx, pendingContext.serialNum, + ref pendingContext.psfUpdateArgs); + break; + case OperationType.PSF_INSERT: + internalStatus = PsfInternalInsert(ref pendingContext.key.Get(), + ref pendingContext.value.Get(), + ref pendingContext.input, ref pendingContext, currentCtx, pendingContext.serialNum); break; case OperationType.DELETE: @@ -1667,23 +1708,22 @@ private bool TraceBackForKeyMatch( long fromLogicalAddress, long minOffset, out long foundLogicalAddress, - out long foundPhysicalAddress) + out long foundPhysicalAddress, + IPSFInput psfInput = null) { foundLogicalAddress = fromLogicalAddress; while (foundLogicalAddress >= minOffset) { foundPhysicalAddress = hlog.GetPhysicalAddress(foundLogicalAddress); - if (comparer.Equals(ref key, ref hlog.GetKey(foundPhysicalAddress))) - { + if (psfInput is null + ? comparer.Equals(ref key, ref hlog.GetKey(foundPhysicalAddress)) + : psfInput.EqualsAt(ref key, ref hlog.GetKey(foundPhysicalAddress))) return true; - } - else - { - foundLogicalAddress = hlog.GetInfo(foundPhysicalAddress).PreviousAddress; - //This makes testing REALLY slow - //Debug.WriteLine("Tracing back"); - continue; - } + + foundLogicalAddress = hlog.GetInfo(foundPhysicalAddress).PreviousAddress; + //This makes testing REALLY slow + //Debug.WriteLine("Tracing back"); + continue; } foundPhysicalAddress = Constants.kInvalidAddress; return false; @@ -1883,7 +1923,8 @@ private long TraceBackForOtherChainStart(long logicalAddress, int bit) #endregion #region Read Cache - private bool ReadFromCache(ref Key key, ref long logicalAddress, ref long physicalAddress, ref int latestRecordVersion) + private bool ReadFromCache(ref Key key, ref long logicalAddress, ref long physicalAddress, + ref int latestRecordVersion, IPSFInput psfInput = null) { HashBucketEntry entry = default; entry.word = logicalAddress; @@ -1894,12 +1935,16 @@ private bool ReadFromCache(ref Key key, ref long logicalAddress, ref long physic while (true) { - if (!readcache.GetInfo(physicalAddress).Invalid && comparer.Equals(ref key, ref readcache.GetKey(physicalAddress))) + if (!readcache.GetInfo(physicalAddress).Invalid) { - if ((logicalAddress & ~Constants.kReadCacheBitMask) >= readcache.SafeReadOnlyAddress) + if (psfInput is null + ? comparer.Equals(ref key, ref readcache.GetKey(physicalAddress)) + : psfInput.EqualsAt(ref key, ref readcache.GetKey(physicalAddress))) { - return true; + if ((logicalAddress & ~Constants.kReadCacheBitMask) >= readcache.SafeReadOnlyAddress) + return true; } + Debug.Assert((logicalAddress & ~Constants.kReadCacheBitMask) >= readcache.SafeHeadAddress); // TODO: copy to tail of read cache // and return new cache entry @@ -1909,11 +1954,31 @@ private bool ReadFromCache(ref Key key, ref long logicalAddress, ref long physic entry.word = logicalAddress; if (!entry.ReadCache) break; physicalAddress = readcache.GetPhysicalAddress(logicalAddress & ~Constants.kReadCacheBitMask); + // TODO: Update latestRecordVersion? } physicalAddress = 0; return false; } + private bool ReadFromCache(ref long logicalAddress, ref long physicalAddress, ref int latestRecordVersion) + { + HashBucketEntry entry = default; + entry.word = logicalAddress; + if (!entry.ReadCache) return false; + + physicalAddress = readcache.GetPhysicalAddress(logicalAddress & ~Constants.kReadCacheBitMask); + latestRecordVersion = readcache.GetInfo(physicalAddress).Version; + + if (!readcache.GetInfo(physicalAddress).Invalid) + { + if ((logicalAddress & ~Constants.kReadCacheBitMask) >= readcache.SafeReadOnlyAddress) + return true; + } + + physicalAddress = 0; + return false; + } + private void SkipReadCache(ref long logicalAddress, ref int latestRecordVersion) { HashBucketEntry entry = default; diff --git a/cs/src/core/Index/FASTER/FASTERThread.cs b/cs/src/core/Index/FASTER/FASTERThread.cs index 483d2332a..8a78d6915 100644 --- a/cs/src/core/Index/FASTER/FASTERThread.cs +++ b/cs/src/core/Index/FASTER/FASTERThread.cs @@ -215,7 +215,7 @@ internal void InternalCompleteRetryRequest(FasterExecutionContext opCtx, FasterE internalStatus = InternalRMW(ref key, ref pendingContext.input, ref pendingContext.userContext, ref pendingContext, currentCtx, pendingContext.serialNum); break; case OperationType.UPSERT: - internalStatus = InternalUpsert(ref key, ref value, ref pendingContext.userContext, ref pendingContext, currentCtx, pendingContext.serialNum); + internalStatus = InternalUpsert(ref key, ref value, ref pendingContext.userContext, ref pendingContext, currentCtx, pendingContext.serialNum, ref pendingContext.psfUpdateArgs); break; case OperationType.DELETE: internalStatus = InternalDelete(ref key, ref pendingContext.userContext, ref pendingContext, currentCtx, pendingContext.serialNum); @@ -226,17 +226,19 @@ internal void InternalCompleteRetryRequest(FasterExecutionContext opCtx, FasterE Status status; - // Handle operation status - if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) + if (internalStatus == OperationStatus.SUCCESS && this.PSFManager.HasPSFs) { - status = (Status)internalStatus; + status = this.PSFManager.Upsert(new FasterKVProviderData(this.hlog, ref key, ref value), + pendingContext.psfUpdateArgs.logicalAddress, pendingContext.psfUpdateArgs.isInserted); } else { - status = HandleOperationStatus(opCtx, currentCtx, pendingContext, internalStatus); + status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : status = HandleOperationStatus(opCtx, currentCtx, pendingContext, internalStatus); } - // If done, callback user code. + // If done, callback user code. if (status == Status.OK || status == Status.NOTFOUND) { if (pendingContext.heldLatch == LatchOperation.Shared) @@ -315,12 +317,15 @@ internal void InternalCompletePendingRequest(FasterExecutionContext opCtx, Faste OperationStatus internalStatus; // Issue the continue command - if (pendingContext.type == OperationType.READ) + if (pendingContext.type == OperationType.READ || + pendingContext.type == OperationType.PSF_READ_KEY || + pendingContext.type == OperationType.PSF_READ_ADDRESS) { internalStatus = InternalContinuePendingRead(opCtx, request, ref pendingContext, currentCtx); } else { + Debug.Assert(pendingContext.type == OperationType.RMW); internalStatus = InternalContinuePendingRMW(opCtx, request, ref pendingContext, currentCtx); ; } diff --git a/cs/src/core/Index/Interfaces/IFasterKV.cs b/cs/src/core/Index/Interfaces/IFasterKV.cs index 2abeaebfa..a982ba129 100644 --- a/cs/src/core/Index/Interfaces/IFasterKV.cs +++ b/cs/src/core/Index/Interfaces/IFasterKV.cs @@ -130,6 +130,104 @@ public interface IFasterKV : IDis #endregion + #region PSF Registration + /// + /// Register a with a simple definition. + /// + /// + /// static TPSFKey? sizePsfFunc(ref TKVKey key, ref TKVValue value) => new TPSFKey(value.size); + /// var sizePsfDef = new FasterKVPSFDefinition{TKVKey, TKVValue, TPSFKey}("sizePSF", sizePsfFunc); + /// var sizePsf = fht.RegisterPSF(sizePsfDef); + /// + /// The type of the key value returned from the + /// A FasterKV-specific form of a PSF definition + /// Optional registration settings for the secondary FasterKV instances, etc. + /// A FasterKV-specific PSF implementation whose TRecordId is long( + PSF RegisterPSF( + FasterKVPSFDefinition def, + PSFRegistrationSettings registrationSettings = null) + where TPSFKey : struct; + + /// + /// Register a with a simple definition. + /// + /// + /// static TPSFKey? sizePsfFunc(ref TKVKey key, ref TKVValue value) => new TPSFKey(value.size); + /// var sizePsfDef = new FasterKVPSFDefinition{TKVKey, TKVValue, TPSFKey}("sizePSF", sizePsfFunc); + /// var sizePsf = fht.RegisterPSF(new [] { sizePsfDef }); + /// + /// The type of the key value returned from the + /// An array of FasterKV-specific forms of PSF definitions + /// Optional registration settings for the secondary FasterKV instances, etc. + /// A FasterKV-specific PSF implementation whose TRecordId is long( + PSF[] RegisterPSF + (FasterKVPSFDefinition[] defs, + PSFRegistrationSettings registrationSettings = null) + where TPSFKey : struct; + + /// + /// Register a with a simple definition. + /// + /// + /// var sizePsf = fht.RegisterPSF("sizePsf", (k, v) => new TPSFKey(v.size)); + /// + /// The type of the key value returned from the + /// The name of the PSF; must be unique across all PSFGroups in this FasterKV instance + /// A Func implementing the PSF, it will be wrapped in a delegate + /// Optional registration settings for the secondary FasterKV instances, etc. + /// A FasterKV-specific implementation whose TRecordId is long( + PSF RegisterPSF( + string psfName, Func psfFunc, + PSFRegistrationSettings registrationSettings = null) + where TPSFKey : struct; + + /// + /// Register multiple with no registration settings. + /// + /// + /// var sizePsf = fht.RegisterPSF(("sizePsf", (k, v) => new TPSFKey(v.size)), + /// ("colorPsf", (k, v) => new TPSFKey(v.color))); + /// + /// The type of the key value returned from the + /// One or more tuples containing a PSF name and implementing Func; the name must be + /// unique across all PSFGroups in this FasterKV instance, and the Func will be wrapped in a delegate + /// "params" won't allow the optional fromAddress and keyComparer, so an overload is provided + /// to specify those + PSF[] RegisterPSF( + params (string, Func)[] psfFuncs) + where TPSFKey : struct; + + /// + /// Register multiple s with registration settings. + /// + /// + /// // Unfortunately the array type cannot be implicitly deduced in current versions of the compiler + /// var sizePsf = fht.RegisterPSF(new (string, Func{TKVKey, TKVValue, TPSFKey>)[] { + /// ("sizePsf", (k, v) => new TPSFKey(v.size)), + /// ("colorPsf", (k, v) => new TPSFKey(v.color))}, + /// keyComparer, fromAddress); + /// + /// The type of the key value returned from the + /// One or more tuples containing a PSF name and implementing Func; the name must be + /// unique across all PSFGroups in this FasterKV instance, and the Func will be wrapped in a delegate + /// Optional registration settings for the secondary FasterKV instances, etc. + /// If the registrationSettings parameters are null, then it is simpler to call the "params" overload + /// than to create the vector explicitly + PSF[] RegisterPSF( + (string, Func)[] psfDefs, + PSFRegistrationSettings registrationSettings = null) + where TPSFKey : struct; + + /// + /// Returns the names of registered s for use in recovery. + /// TODO: Supplement or replace this with an app version string. + /// + /// An array of string arrays; each outer array corresponds to a + /// + string[][] GetRegisteredPSFs(); + + #endregion PSF Registration + #region Growth and Recovery /// diff --git a/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs b/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs new file mode 100644 index 000000000..2f8cb1aca --- /dev/null +++ b/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; + +namespace FASTER.core +{ + /// + /// The definition of a single PSF (Predicate Subset Function) + /// + /// The type of the key in the primary FasterKV instance + /// The type of the value in the primary FasterKV instance + /// The type of the key returned by the Predicate and store in the secondary + /// (PSF-implementing) FasterKV instances + public class FasterKVPSFDefinition : IPSFDefinition, TPSFKey> + where TKVKey : new() + where TKVValue : new() + where TPSFKey : struct + { + /// + /// The definition of the delegate used to obtain a new key matching the Value for this PSF definition. + /// + /// The key sent to FasterKV on Upsert or RMW + /// The value sent to FasterKV on Upsert or RMW + /// This must be a delegate instead of a lambda to allow ref parameters + /// Null if the value does not match the predicate, else a key for the value in the PSF hash table + public delegate TPSFKey? PredicateFunc(ref TKVKey kvKey, ref TKVValue kvValue); + + /// + /// The predicate function that will be called by FasterKV on Upsert or RMW. + /// + public PredicateFunc Predicate; + + /// + /// Executes the Predicate + /// + /// The record obtained from the primary FasterKV instance + /// + /// Null if the value does not match the predicate, else a key for the value in the PSF hash table + public TPSFKey? Execute(FasterKVProviderData record) + => Predicate(ref record.GetKey(), ref record.GetValue()); + + /// + /// The Name of the PSF, assigned by the caller. Must be unique among all PSFs (TODO: enforce). + /// + public string Name { get; } + + /// + /// Instantiates the instance with the name and predicate delegate + /// + /// + /// + public FasterKVPSFDefinition(string name, PredicateFunc predicate) + { + this.Name = name; + this.Predicate = predicate; + } + + /// + /// Instantiates the instance with the name and predicate Func{}, which we wrap in a delegate. + /// This allows a streamlined API call. + /// + /// + /// + public FasterKVPSFDefinition(string name, Func predicate) + { + TPSFKey? wrappedPredicate(ref TKVKey key, ref TKVValue value) => predicate(key, value); + + this.Name = name; + this.Predicate = wrappedPredicate; + } + } +} + \ No newline at end of file diff --git a/cs/src/core/Index/PSF/FasterKVProviderData.cs b/cs/src/core/Index/PSF/FasterKVProviderData.cs new file mode 100644 index 000000000..40e064ec5 --- /dev/null +++ b/cs/src/core/Index/PSF/FasterKVProviderData.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + /// + /// The wrapper around the provider data stored in the primary faster instance. + /// + /// The type of the key in the primary FasterKV instance + /// The type of the value in the primary FasterKV instance + /// Having this enables separation between the LogicalAddress stored in the PSF-implementing + /// FasterKV instances, and the actual and + /// types. + public class FasterKVProviderData : IDisposable + where TKVKey : new() + where TKVValue : new() + { + // C# doesn't allow ref fields and even if it did, if the client held the FasterKVProviderData + // past the ref lifetime, bad things would happen when accessing the ref key/value. + internal IHeapContainer keyContainer; // TODO: Perf efficiency + internal IHeapContainer valueContainer; + + internal FasterKVProviderData(AllocatorBase allocator, ref TKVKey key, ref TKVValue value) + { + this.keyContainer = allocator.GetKeyContainer(ref key); + this.valueContainer = allocator.GetValueContainer(ref value); + } + + public unsafe ref TKVKey GetKey() => ref this.keyContainer.Get(); + + public unsafe ref TKVValue GetValue() => ref this.valueContainer.Get(); + + /// + public void Dispose() + { + this.keyContainer.Dispose(); + this.valueContainer.Dispose(); + } + } +} diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs new file mode 100644 index 000000000..deae742ce --- /dev/null +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -0,0 +1,1063 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + // PSF-related internal function implementations for FasterKV; these correspond to the similarly-named + // functions in FasterImpl.cs. + public unsafe partial class FasterKV + : FasterBase, IFasterKV + where Key : new() + where Value : new() + where Functions : IFunctions + { + internal IChainPost chainPost; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal OperationStatus PsfInternalReadKey( + ref Key queryKey, ref PSFReadArgs psfArgs, + ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) + { + var bucket = default(HashBucket*); + var slot = default(int); + var physicalAddress = default(long); + var latestRecordVersion = -1; + var heldOperation = LatchOperation.None; + + var psfInput = psfArgs.Input; + var psfOutput = psfArgs.Output; + + var hash = psfInput.GetHashCode64At(ref queryKey); + var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); + + if (sessionCtx.phase != Phase.REST) + HeavyEnter(hash, sessionCtx); + + #region Trace back for record in in-memory HybridLog + HashBucketEntry entry = default; + var tagExists = FindTag(hash, tag, ref bucket, ref slot, ref entry); + OperationStatus status; + long logicalAddress; + if (tagExists) + { + logicalAddress = entry.Address; + + if (UseReadCache && ReadFromCache(ref queryKey, ref logicalAddress, ref physicalAddress, ref latestRecordVersion, psfInput)) + { + if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot thread + } + return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref readcache.GetValue(physicalAddress), isConcurrent: false).Status; + } + + if (logicalAddress >= hlog.HeadAddress) + { + physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + if (latestRecordVersion == -1) + latestRecordVersion = hlog.GetInfo(physicalAddress).Version; + + if (!psfInput.EqualsAt(ref queryKey, ref hlog.GetKey(physicalAddress))) + { + logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; + TraceBackForKeyMatch(ref queryKey, + logicalAddress, + hlog.HeadAddress, + out logicalAddress, + out physicalAddress, + psfInput); + } + } + } + else + { + // no tag found + return OperationStatus.NOTFOUND; + } + #endregion + + if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot thread + } + + #region Normal processing + + // Mutable region (even fuzzy region is included here) + if (logicalAddress >= hlog.SafeReadOnlyAddress) + { + return hlog.GetInfo(physicalAddress).Tombstone + ? OperationStatus.NOTFOUND + : psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref hlog.GetValue(physicalAddress), isConcurrent:true).Status; + } + + // Immutable region + else if (logicalAddress >= hlog.HeadAddress) + { + return hlog.GetInfo(physicalAddress).Tombstone + ? OperationStatus.NOTFOUND + : psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref hlog.GetValue(physicalAddress), isConcurrent:true).Status; + } + + // On-Disk Region + else if (logicalAddress >= hlog.BeginAddress) + { + status = OperationStatus.RECORD_ON_DISK; + if (sessionCtx.phase == Phase.PREPARE) + { + Debug.Assert(heldOperation != LatchOperation.Exclusive); + if (heldOperation == LatchOperation.Shared || HashBucket.TryAcquireSharedLatch(bucket)) + heldOperation = LatchOperation.Shared; + else + status = OperationStatus.CPR_SHIFT_DETECTED; + + if (RelaxedCPR) // don't hold on to shared latched during IO + { + if (heldOperation == LatchOperation.Shared) + HashBucket.ReleaseSharedLatch(bucket); + heldOperation = LatchOperation.None; + } + } + goto CreatePendingContext; + } + else + { + // No record found + return OperationStatus.NOTFOUND; + } + + #endregion + + #region Create pending context + CreatePendingContext: + { + pendingContext.type = OperationType.PSF_READ_KEY; + pendingContext.key = hlog.GetKeyContainer(ref queryKey); + pendingContext.input = default; + pendingContext.output = default; + pendingContext.userContext = default; + pendingContext.entry.word = entry.word; + pendingContext.logicalAddress = logicalAddress; + pendingContext.version = sessionCtx.version; + pendingContext.serialNum = lsn; + pendingContext.heldLatch = heldOperation; + pendingContext.psfReadArgs = psfArgs; + } + #endregion + + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal OperationStatus PsfInternalReadAddress( + ref PSFReadArgs psfArgs, + ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) + { + var physicalAddress = default(long); + var latestRecordVersion = -1; + + var psfInput = psfArgs.Input; + var psfOutput = psfArgs.Output; + + // TODO: For the primary FasterKV, we do not have any queryKey here to get hash and tag and do latching; + // verify this wrt RelaxedCRT + // TODO: Verify we should always find the LogicalAddress + OperationStatus status; + + #region Look up record in in-memory HybridLog + long logicalAddress = psfInput.ReadLogicalAddress; + if (UseReadCache && ReadFromCache(ref logicalAddress, ref physicalAddress, ref latestRecordVersion)) + { + if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot thread + } + return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref readcache.GetValue(physicalAddress), isConcurrent: false).Status; + } + + if (logicalAddress >= hlog.HeadAddress) + { + physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + if (latestRecordVersion == -1) + latestRecordVersion = hlog.GetInfo(physicalAddress).Version; + } + #endregion + + if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot thread + } + + #region Normal processing + + // Mutable region (even fuzzy region is included here) + if (logicalAddress >= hlog.SafeReadOnlyAddress) + { + return hlog.GetInfo(physicalAddress).Tombstone + ? OperationStatus.NOTFOUND + : psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref hlog.GetValue(physicalAddress), isConcurrent: true).Status; + } + + // Immutable region + else if (logicalAddress >= hlog.HeadAddress) + { + return hlog.GetInfo(physicalAddress).Tombstone + ? OperationStatus.NOTFOUND + : psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref hlog.GetValue(physicalAddress), isConcurrent: true).Status; + } + + // On-Disk Region + else if (logicalAddress >= hlog.BeginAddress) + { + status = OperationStatus.RECORD_ON_DISK; + goto CreatePendingContext; + } + else + { + // No record found + return OperationStatus.NOTFOUND; + } + + #endregion + + #region Create pending context + CreatePendingContext: + { + pendingContext.type = OperationType.PSF_READ_ADDRESS; + pendingContext.key = default; + pendingContext.input = default; + pendingContext.output = default; + pendingContext.userContext = default; + pendingContext.entry.word = default; + pendingContext.logicalAddress = logicalAddress; + pendingContext.version = sessionCtx.version; + pendingContext.serialNum = lsn; + pendingContext.heldLatch = LatchOperation.None; + pendingContext.psfReadArgs = psfArgs; + } + #endregion + + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal OperationStatus PsfInternalInsert( + ref Key compositeKey, ref Value value, ref Input input, + ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) + { + var status = default(OperationStatus); + var bucket = default(HashBucket*); + var slot = default(int); + var logicalAddress = Constants.kInvalidAddress; + var physicalAddress = default(long); + var latchOperation = default(LatchOperation); + var latestRecordVersion = -1; + var entry = default(HashBucketEntry); + + var psfInput = input as IPSFInput; + + // Update the PSFValue links for chains with nullIndicator false (indicating a match with the + // corresponding PSF) to point to the previous records for all keys in the composite key. + // TODO: We're not checking for a previous occurrence of the PSFValue's recordId because + // we are doing insert only here; the update part of upsert is done in PsfInternalRMW. + var chainHeight = this.chainPost.ChainHeight; + long* hashes = stackalloc long[chainHeight]; + long* chainLinkPtrs = this.chainPost.GetChainLinkPtrs(ref value); + for (var chainLinkIdx = 0; chainLinkIdx < chainHeight; ++chainLinkIdx) + { + psfInput.PsfOrdinal = chainLinkIdx; + if (psfInput.IsNullAt) + continue; + long* chainLinkPtr = chainLinkPtrs + chainLinkIdx; + + var hash = psfInput.GetHashCode64At(ref compositeKey); + *(hashes + chainLinkIdx) = hash; + var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); + + if (sessionCtx.phase != Phase.REST) + HeavyEnter(hash, sessionCtx); + + #region Trace back for record in in-memory HybridLog + entry = default; + var tagExists = FindTag(hash, tag, ref bucket, ref slot, ref entry); + if (tagExists) + { + logicalAddress = entry.Address; + + // TODO: is this needed? If so, verify handling for multiple keys (should abandon here and restart on retry) + if (UseReadCache && ReadFromCache(ref compositeKey, ref logicalAddress, ref physicalAddress, ref latestRecordVersion)) + { + if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot thread + } + } + + if (logicalAddress >= hlog.HeadAddress) + { + physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + if (latestRecordVersion == -1) + latestRecordVersion = hlog.GetInfo(physicalAddress).Version; + + + if (!psfInput.EqualsAt(ref compositeKey, ref hlog.GetKey(physicalAddress))) + { + logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; + TraceBackForKeyMatch(ref compositeKey, + logicalAddress, + hlog.HeadAddress, + out logicalAddress, + out physicalAddress, + psfInput); + } + } + + if (!hlog.GetInfo(physicalAddress).Tombstone) + *chainLinkPtr = logicalAddress; + } + #endregion + } + + #region Entry latch operation + if (sessionCtx.phase != Phase.REST) + { + switch (sessionCtx.phase) + { + case Phase.PREPARE: + { + if (HashBucket.TryAcquireSharedLatch(bucket)) + { + // Set to release shared latch (default) + latchOperation = LatchOperation.Shared; + if (latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot Thread + } + break; // Normal Processing + } + else + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot Thread + } + } + case Phase.IN_PROGRESS: + { + if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) + { + if (HashBucket.TryAcquireExclusiveLatch(bucket)) + { + // Set to release exclusive latch (default) + latchOperation = LatchOperation.Exclusive; + goto CreateNewRecord; // Create a (v+1) record + } + else + { + status = OperationStatus.RETRY_LATER; + goto CreatePendingContext; // Go Pending + } + } + break; // Normal Processing + } + case Phase.WAIT_PENDING: + { + if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) + { + if (HashBucket.NoSharedLatches(bucket)) + { + goto CreateNewRecord; // Create a (v+1) record + } + else + { + status = OperationStatus.RETRY_LATER; + goto CreatePendingContext; // Go Pending + } + } + break; // Normal Processing + } + case Phase.WAIT_FLUSH: + { + if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) + { + goto CreateNewRecord; // Create a (v+1) record + } + break; // Normal Processing + } + default: + break; + } + } + #endregion + + Debug.Assert(latestRecordVersion <= sessionCtx.version); + + #region Create new record in the mutable region + CreateNewRecord: + { + // Immutable region or new record + var recordSize = hlog.GetRecordSize(ref compositeKey, ref value); + BlockAllocate(recordSize, out long newLogicalAddress, sessionCtx); + var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); + RecordInfo.WriteInfo(ref hlog.GetInfo(newPhysicalAddress), sessionCtx.version, + final:true, tombstone:false, invalidBit:false, + Constants.kInvalidAddress); // We manage all prev addresses within PSFValue + hlog.ShallowCopy(ref compositeKey, ref hlog.GetKey(newPhysicalAddress)); + functions.SingleWriter(ref compositeKey, ref value, ref hlog.GetValue(newPhysicalAddress)); + + for (var chainLinkIdx = 0; chainLinkIdx < chainHeight; ++chainLinkIdx) + { + psfInput.PsfOrdinal = chainLinkIdx; + if (psfInput.IsNullAt) + continue; + + var hash = *(hashes + chainLinkIdx); + var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); + entry = default; + FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry, hlog.BeginAddress); + + var updatedEntry = default(HashBucketEntry); + updatedEntry.Tag = tag; + updatedEntry.Address = newLogicalAddress & Constants.kAddressMask; + updatedEntry.Pending = entry.Pending; + updatedEntry.Tentative = false; + + var foundEntry = default(HashBucketEntry); + foundEntry.word = Interlocked.CompareExchange(ref bucket->bucket_entries[slot], + updatedEntry.word, entry.word); + + if (foundEntry.word != entry.word) + { + hlog.GetInfo(newPhysicalAddress).Invalid = true; + status = OperationStatus.RETRY_NOW; + goto LatchRelease; + } + } + + status = OperationStatus.SUCCESS; + goto LatchRelease; + } + #endregion + + #region Create pending context + CreatePendingContext: + { + psfInput.PsfOrdinal = Constants.kInvalidPsfOrdinal; + + pendingContext.type = OperationType.PSF_INSERT; + pendingContext.key = hlog.GetKeyContainer(ref compositeKey); + pendingContext.value = hlog.GetValueContainer(ref value); + pendingContext.input = input; + pendingContext.userContext = default; + pendingContext.entry.word = entry.word; + pendingContext.logicalAddress = logicalAddress; + pendingContext.version = sessionCtx.version; + pendingContext.serialNum = lsn; + } + #endregion + + #region Latch release + LatchRelease: + { + switch (latchOperation) + { + case LatchOperation.Shared: + HashBucket.ReleaseSharedLatch(bucket); + break; + case LatchOperation.Exclusive: + HashBucket.ReleaseExclusiveLatch(bucket); + break; + default: + break; + } + } + #endregion + + return status == OperationStatus.RETRY_NOW + ? PsfInternalInsert(ref compositeKey, ref value, ref input, ref pendingContext, sessionCtx, lsn) + : status; + } + + internal OperationStatus PsfInternalRMW( // TODO + ref Key key, ref Input input, + ref Context userContext, + ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) + { + var recordSize = default(int); + var bucket = default(HashBucket*); + var slot = default(int); + var logicalAddress = Constants.kInvalidAddress; + var physicalAddress = default(long); + var latestRecordVersion = -1; + var status = default(OperationStatus); + var latchOperation = LatchOperation.None; + var heldOperation = LatchOperation.None; + + var hash = comparer.GetHashCode64(ref key); + var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); + + if (sessionCtx.phase != Phase.REST) + HeavyEnter(hash, sessionCtx); + + #region Trace back for record in in-memory HybridLog + var entry = default(HashBucketEntry); + FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry, hlog.BeginAddress); + logicalAddress = entry.Address; + + // For simplicity, we don't let RMW operations use read cache + if (UseReadCache) + SkipReadCache(ref logicalAddress, ref latestRecordVersion); + var latestLogicalAddress = logicalAddress; + + if (logicalAddress >= hlog.HeadAddress) + { + physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + latestRecordVersion = hlog.GetInfo(physicalAddress).Version; + + if (!comparer.Equals(ref key, ref hlog.GetKey(physicalAddress))) + { + logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; + TraceBackForKeyMatch(ref key, logicalAddress, + hlog.HeadAddress, + out logicalAddress, + out physicalAddress); + } + } + #endregion + + // Optimization for the most common case + if (sessionCtx.phase == Phase.REST && logicalAddress >= hlog.ReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) + { + if (functions.InPlaceUpdater(ref key, ref input, ref hlog.GetValue(physicalAddress))) + { + return OperationStatus.SUCCESS; + } + } + + #region Entry latch operation + if (sessionCtx.phase != Phase.REST) + { + switch (sessionCtx.phase) + { + case Phase.PREPARE: + { + Debug.Assert(pendingContext.heldLatch != LatchOperation.Exclusive); + if (pendingContext.heldLatch == LatchOperation.Shared || HashBucket.TryAcquireSharedLatch(bucket)) + { + // Set to release shared latch (default) + latchOperation = LatchOperation.Shared; + if (latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreateFailureContext; // Pivot Thread + } + break; // Normal Processing + } + else + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreateFailureContext; // Pivot Thread + } + } + case Phase.IN_PROGRESS: + { + if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) + { + Debug.Assert(pendingContext.heldLatch != LatchOperation.Shared); + if (pendingContext.heldLatch == LatchOperation.Exclusive || HashBucket.TryAcquireExclusiveLatch(bucket)) + { + // Set to release exclusive latch (default) + latchOperation = LatchOperation.Exclusive; + goto CreateNewRecord; // Create a (v+1) record + } + else + { + status = OperationStatus.RETRY_LATER; + goto CreateFailureContext; // Go Pending + } + } + break; // Normal Processing + } + case Phase.WAIT_PENDING: + { + if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) + { + if (HashBucket.NoSharedLatches(bucket)) + { + goto CreateNewRecord; // Create a (v+1) record + } + else + { + status = OperationStatus.RETRY_LATER; + goto CreateFailureContext; // Go Pending + } + } + break; // Normal Processing + } + case Phase.WAIT_FLUSH: + { + if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) + { + goto CreateNewRecord; // Create a (v+1) record + } + break; // Normal Processing + } + default: + break; + } + } + #endregion + + Debug.Assert(latestRecordVersion <= sessionCtx.version); + + #region Normal processing + + // Mutable Region: Update the record in-place + if (logicalAddress >= hlog.ReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) + { + if (FoldOverSnapshot) + { + Debug.Assert(hlog.GetInfo(physicalAddress).Version == sessionCtx.version); + } + + if (functions.InPlaceUpdater(ref key, ref input, ref hlog.GetValue(physicalAddress))) + { + status = OperationStatus.SUCCESS; + goto LatchRelease; // Release shared latch (if acquired) + } + } + + // Fuzzy Region: Must go pending due to lost-update anomaly + else if (logicalAddress >= hlog.SafeReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) + { + status = OperationStatus.RETRY_LATER; + // Do not retain latch for pendings ops in relaxed CPR + if (!RelaxedCPR) + { + // Retain the shared latch (if acquired) + if (latchOperation == LatchOperation.Shared) + { + heldOperation = latchOperation; + latchOperation = LatchOperation.None; + } + } + goto CreateFailureContext; // Go pending + } + + // Safe Read-Only Region: Create a record in the mutable region + else if (logicalAddress >= hlog.HeadAddress) + { + goto CreateNewRecord; + } + + // Disk Region: Need to issue async io requests + else if (logicalAddress >= hlog.BeginAddress) + { + status = OperationStatus.RECORD_ON_DISK; + // Do not retain latch for pendings ops in relaxed CPR + if (!RelaxedCPR) + { + // Retain the shared latch (if acquired) + if (latchOperation == LatchOperation.Shared) + { + heldOperation = latchOperation; + latchOperation = LatchOperation.None; + } + } + goto CreateFailureContext; // Go pending + } + + // No record exists - create new + else + { + goto CreateNewRecord; + } + + #endregion + + #region Create new record + CreateNewRecord: + { + recordSize = (logicalAddress < hlog.BeginAddress) ? + hlog.GetInitialRecordSize(ref key, ref input) : + hlog.GetRecordSize(physicalAddress); + BlockAllocate(recordSize, out long newLogicalAddress, sessionCtx); + var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); + RecordInfo.WriteInfo(ref hlog.GetInfo(newPhysicalAddress), sessionCtx.version, + true, false, false, + latestLogicalAddress); + hlog.ShallowCopy(ref key, ref hlog.GetKey(newPhysicalAddress)); + if (logicalAddress < hlog.BeginAddress) + { + functions.InitialUpdater(ref key, ref input, ref hlog.GetValue(newPhysicalAddress)); + status = OperationStatus.NOTFOUND; + } + else if (logicalAddress >= hlog.HeadAddress) + { + if (hlog.GetInfo(physicalAddress).Tombstone) + { + functions.InitialUpdater(ref key, ref input, ref hlog.GetValue(newPhysicalAddress)); + status = OperationStatus.NOTFOUND; + } + else + { + functions.CopyUpdater(ref key, ref input, + ref hlog.GetValue(physicalAddress), + ref hlog.GetValue(newPhysicalAddress)); + status = OperationStatus.SUCCESS; + } + } + else + { + // ah, old record slipped onto disk + hlog.GetInfo(newPhysicalAddress).Invalid = true; + status = OperationStatus.RETRY_NOW; + goto LatchRelease; + } + + var updatedEntry = default(HashBucketEntry); + updatedEntry.Tag = tag; + updatedEntry.Address = newLogicalAddress & Constants.kAddressMask; + updatedEntry.Pending = entry.Pending; + updatedEntry.Tentative = false; + + var foundEntry = default(HashBucketEntry); + foundEntry.word = Interlocked.CompareExchange( + ref bucket->bucket_entries[slot], + updatedEntry.word, entry.word); + + if (foundEntry.word == entry.word) + { + goto LatchRelease; + } + else + { + // CAS failed + hlog.GetInfo(newPhysicalAddress).Invalid = true; + status = OperationStatus.RETRY_NOW; + goto LatchRelease; + } + } + #endregion + + #region Create failure context + CreateFailureContext: + { + pendingContext.type = OperationType.RMW; + pendingContext.key = hlog.GetKeyContainer(ref key); + pendingContext.input = input; + pendingContext.userContext = userContext; + pendingContext.entry.word = entry.word; + pendingContext.logicalAddress = logicalAddress; + pendingContext.version = sessionCtx.version; + pendingContext.serialNum = lsn; + pendingContext.heldLatch = heldOperation; + } + #endregion + + #region Latch release + LatchRelease: + { + switch (latchOperation) + { + case LatchOperation.Shared: + HashBucket.ReleaseSharedLatch(bucket); + break; + case LatchOperation.Exclusive: + HashBucket.ReleaseExclusiveLatch(bucket); + break; + default: + break; + } + } + #endregion + + return status == OperationStatus.RETRY_NOW + ? PsfInternalRMW(ref key, ref input, ref userContext, ref pendingContext, sessionCtx, lsn) + : status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal OperationStatus PsfInternalDelete( // TODO + ref Key key, + ref Context userContext, + ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) + { + var status = default(OperationStatus); + var bucket = default(HashBucket*); + var slot = default(int); + var logicalAddress = Constants.kInvalidAddress; + var physicalAddress = default(long); + var latchOperation = default(LatchOperation); + var version = default(int); + var latestRecordVersion = -1; + + var hash = comparer.GetHashCode64(ref key); + var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); + + if (sessionCtx.phase != Phase.REST) + HeavyEnter(hash, sessionCtx); + + #region Trace back for record in in-memory HybridLog + var entry = default(HashBucketEntry); + var tagExists = FindTag(hash, tag, ref bucket, ref slot, ref entry); + if (!tagExists) + return OperationStatus.NOTFOUND; + + logicalAddress = entry.Address; + + if (UseReadCache) + SkipAndInvalidateReadCache(ref logicalAddress, ref latestRecordVersion, ref key); + var latestLogicalAddress = logicalAddress; + + if (logicalAddress >= hlog.ReadOnlyAddress) + { + physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + if (latestRecordVersion == -1) + latestRecordVersion = hlog.GetInfo(physicalAddress).Version; + if (!comparer.Equals(ref key, ref hlog.GetKey(physicalAddress))) + { + logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; + TraceBackForKeyMatch(ref key, + logicalAddress, + hlog.ReadOnlyAddress, + out logicalAddress, + out physicalAddress); + } + } + #endregion + + // NO optimization for most common case + //if (sessionCtx.phase == Phase.REST && logicalAddress >= hlog.ReadOnlyAddress) + //{ + // hlog.GetInfo(physicalAddress).Tombstone = true; + // return OperationStatus.SUCCESS; + //} + + #region Entry latch operation + if (sessionCtx.phase != Phase.REST) + { + switch (sessionCtx.phase) + { + case Phase.PREPARE: + { + version = sessionCtx.version; + if (HashBucket.TryAcquireSharedLatch(bucket)) + { + // Set to release shared latch (default) + latchOperation = LatchOperation.Shared; + if (latestRecordVersion != -1 && latestRecordVersion > version) + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot Thread + } + break; // Normal Processing + } + else + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot Thread + } + } + case Phase.IN_PROGRESS: + { + version = (sessionCtx.version - 1); + if (latestRecordVersion != -1 && latestRecordVersion <= version) + { + if (HashBucket.TryAcquireExclusiveLatch(bucket)) + { + // Set to release exclusive latch (default) + latchOperation = LatchOperation.Exclusive; + goto CreateNewRecord; // Create a (v+1) record + } + else + { + status = OperationStatus.RETRY_LATER; + goto CreatePendingContext; // Go Pending + } + } + break; // Normal Processing + } + case Phase.WAIT_PENDING: + { + version = (sessionCtx.version - 1); + if (latestRecordVersion != -1 && latestRecordVersion <= version) + { + if (HashBucket.NoSharedLatches(bucket)) + { + goto CreateNewRecord; // Create a (v+1) record + } + else + { + status = OperationStatus.RETRY_LATER; + goto CreatePendingContext; // Go Pending + } + } + break; // Normal Processing + } + case Phase.WAIT_FLUSH: + { + version = (sessionCtx.version - 1); + if (latestRecordVersion != -1 && latestRecordVersion <= version) + { + goto CreateNewRecord; // Create a (v+1) record + } + break; // Normal Processing + } + default: + break; + } + } + #endregion + + Debug.Assert(latestRecordVersion <= sessionCtx.version); + + #region Normal processing + + // Record is in memory, try to update hash chain and completely elide record + // only if previous address points to invalid address + if (logicalAddress >= hlog.ReadOnlyAddress) + { + if (entry.Address == logicalAddress && hlog.GetInfo(physicalAddress).PreviousAddress < hlog.BeginAddress) + { + var updatedEntry = default(HashBucketEntry); + updatedEntry.Tag = 0; + if (hlog.GetInfo(physicalAddress).PreviousAddress == Constants.kTempInvalidAddress) + updatedEntry.Address = Constants.kInvalidAddress; + else + updatedEntry.Address = hlog.GetInfo(physicalAddress).PreviousAddress; + updatedEntry.Pending = entry.Pending; + updatedEntry.Tentative = false; + + if (entry.word == Interlocked.CompareExchange(ref bucket->bucket_entries[slot], updatedEntry.word, entry.word)) + { + // Apply tombstone bit to the record + hlog.GetInfo(physicalAddress).Tombstone = true; + + if (WriteDefaultOnDelete) + { + // Write default value + // Ignore return value, the record is already marked + Value v = default; + functions.ConcurrentWriter(ref hlog.GetKey(physicalAddress), ref v, ref hlog.GetValue(physicalAddress)); + } + + status = OperationStatus.SUCCESS; + goto LatchRelease; // Release shared latch (if acquired) + } + } + } + + // Mutable Region: Update the record in-place + if (logicalAddress >= hlog.ReadOnlyAddress) + { + hlog.GetInfo(physicalAddress).Tombstone = true; + + if (WriteDefaultOnDelete) + { + // Write default value + // Ignore return value, the record is already marked + Value v = default; + functions.ConcurrentWriter(ref hlog.GetKey(physicalAddress), ref v, ref hlog.GetValue(physicalAddress)); + } + + status = OperationStatus.SUCCESS; + goto LatchRelease; // Release shared latch (if acquired) + } + + // All other regions: Create a record in the mutable region + #endregion + + #region Create new record in the mutable region + CreateNewRecord: + { + var value = default(Value); + // Immutable region or new record + // Allocate default record size for tombstone + var recordSize = hlog.GetRecordSize(ref key, ref value); + BlockAllocate(recordSize, out long newLogicalAddress, sessionCtx); + var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); + RecordInfo.WriteInfo(ref hlog.GetInfo(newPhysicalAddress), + sessionCtx.version, + true, true, false, + latestLogicalAddress); + hlog.ShallowCopy(ref key, ref hlog.GetKey(newPhysicalAddress)); + + var updatedEntry = default(HashBucketEntry); + updatedEntry.Tag = tag; + updatedEntry.Address = newLogicalAddress & Constants.kAddressMask; + updatedEntry.Pending = entry.Pending; + updatedEntry.Tentative = false; + + var foundEntry = default(HashBucketEntry); + foundEntry.word = Interlocked.CompareExchange( + ref bucket->bucket_entries[slot], + updatedEntry.word, entry.word); + + if (foundEntry.word == entry.word) + { + status = OperationStatus.SUCCESS; + goto LatchRelease; + } + else + { + hlog.GetInfo(newPhysicalAddress).Invalid = true; + status = OperationStatus.RETRY_NOW; + goto LatchRelease; + } + } + #endregion + + #region Create pending context + CreatePendingContext: + { + pendingContext.type = OperationType.DELETE; + pendingContext.key = hlog.GetKeyContainer(ref key); + pendingContext.userContext = userContext; + pendingContext.entry.word = entry.word; + pendingContext.logicalAddress = logicalAddress; + pendingContext.version = sessionCtx.version; + pendingContext.serialNum = lsn; + } + #endregion + + #region Latch release + LatchRelease: + { + switch (latchOperation) + { + case LatchOperation.Shared: + HashBucket.ReleaseSharedLatch(bucket); + break; + case LatchOperation.Exclusive: + HashBucket.ReleaseExclusiveLatch(bucket); + break; + default: + break; + } + } + #endregion + + return status == OperationStatus.RETRY_NOW + ? PsfInternalDelete(ref key, ref userContext, ref pendingContext, sessionCtx, lsn) + : status; + } + } +} \ No newline at end of file diff --git a/cs/src/core/Index/PSF/FasterPSFRegistration.cs b/cs/src/core/Index/PSF/FasterPSFRegistration.cs new file mode 100644 index 000000000..f5bb36905 --- /dev/null +++ b/cs/src/core/Index/PSF/FasterPSFRegistration.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Linq; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + // PSF-related functions for FasterKV + public partial class FasterKV + : FasterBase, IFasterKV + where Key : new() + where Value : new() + where Functions : IFunctions + { + // Some FasterKV ctor params we need to pass through. + private readonly long hashTableSize; + private readonly LogSettings logSettings; + private readonly CheckpointSettings checkpointSettings; + private readonly SerializerSettings serializerSettings; + private readonly VariableLengthStructSettings variableLengthStructSettings; + + internal PSFManager, long> PSFManager { get; private set; } + + internal void InitializePSFs() + => this.PSFManager = new PSFManager, long>(); + + private PSFRegistrationSettings CreateDefaultRegistrationSettings() + => new PSFRegistrationSettings + { + HashTableSize = this.hashTableSize, + LogSettings = this.logSettings, + CheckpointSettings = this.checkpointSettings // TODO fix this up + }; + + #region PSF Registration API + /// + public PSF RegisterPSF( + FasterKVPSFDefinition def, + PSFRegistrationSettings registrationSettings = null) + where TPSFKey : struct + => this.PSFManager.RegisterPSF(def, registrationSettings ?? CreateDefaultRegistrationSettings()); + + /// + public PSF[] RegisterPSF + (FasterKVPSFDefinition[] defs, + PSFRegistrationSettings registrationSettings = null) + where TPSFKey : struct + => this.PSFManager.RegisterPSF(defs, registrationSettings ?? CreateDefaultRegistrationSettings()); + + /// + public PSF RegisterPSF( + string psfName, Func psfFunc, + PSFRegistrationSettings registrationSettings = null) + where TPSFKey : struct + => this.PSFManager.RegisterPSF(new FasterKVPSFDefinition(psfName, psfFunc), + registrationSettings ?? CreateDefaultRegistrationSettings()); + + /// + public PSF[] RegisterPSF( + params (string, Func)[] psfFuncs) + where TPSFKey : struct + => this.PSFManager.RegisterPSF(psfFuncs.Select(e => new FasterKVPSFDefinition(e.Item1, e.Item2)).ToArray(), + CreateDefaultRegistrationSettings()); + + /// + public PSF[] RegisterPSF( + (string, Func)[] psfDefs, + PSFRegistrationSettings registrationSettings = null) + where TPSFKey : struct + => this.PSFManager.RegisterPSF(psfDefs.Select(e => new FasterKVPSFDefinition(e.Item1, e.Item2)).ToArray(), + CreateDefaultRegistrationSettings()); + + /// + public string[][] GetRegisteredPSFs() => this.PSFManager.GetRegisteredPSFs(); + #endregion PSF Registration API + } +} diff --git a/cs/src/core/Index/PSF/IChainPost.cs b/cs/src/core/Index/PSF/IChainPost.cs new file mode 100644 index 000000000..365d6b216 --- /dev/null +++ b/cs/src/core/Index/PSF/IChainPost.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + internal unsafe interface IChainPost + { + int ChainHeight { get; } + + int RecordIdSize { get; } + + long* GetChainLinkPtrs(ref TPSFValue value); + } +} \ No newline at end of file diff --git a/cs/src/core/Index/PSF/IExecutePSF.cs b/cs/src/core/Index/PSF/IExecutePSF.cs new file mode 100644 index 000000000..c580d0ba6 --- /dev/null +++ b/cs/src/core/Index/PSF/IExecutePSF.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; + +namespace FASTER.core +{ + /// + /// This interface is implemented on a and decouples + /// the execution from the knowledge of the TKVKey and TKVValue of the + /// primary FasterKV instance. + /// + /// + /// + public interface IExecutePSF + { + /// + /// For each in the , + /// and store the resultant TPSFKey in the secondary FasterKV instance. + /// + /// The provider's data, e.g. + /// The provider's record ID, e.g. long (logicalAddress) for FasterKV + /// + Status ExecuteAndStore(TProviderData data, TRecordId recordId, bool isInserted); + + /// + /// For the given , verify that the + /// matches (returns non-null). + /// + /// The provider data wrapped around the TRecordId + /// The ordinal of the in the + /// . + /// True if the providerData matches (returns non-null from) the , + /// else false. + bool Verify(TProviderData providerData, int psfOrdinal); + } +} diff --git a/cs/src/core/Index/PSF/IPSF.cs b/cs/src/core/Index/PSF/IPSF.cs new file mode 100644 index 000000000..fc9f59a29 --- /dev/null +++ b/cs/src/core/Index/PSF/IPSF.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// A base interface for to decouple the generic type parameters. + /// + public interface IPSF + { + /// + /// The name of the ; must be unique among all // TODO ensure this + /// s. + /// + string Name { get; } + } +} diff --git a/cs/src/core/Index/PSF/IPSFCreateProviderData.cs b/cs/src/core/Index/PSF/IPSFCreateProviderData.cs new file mode 100644 index 000000000..aecf4e275 --- /dev/null +++ b/cs/src/core/Index/PSF/IPSFCreateProviderData.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// The data provider's instantiator that takes the RecordId and creates the data class the user sees. + /// + public interface IPSFCreateProviderData + { + /// + /// Creates the provider data from the record id. + /// + TProviderData Create(TRecordId recordId); + } +} diff --git a/cs/src/core/Index/PSF/IPSFDefinition.cs b/cs/src/core/Index/PSF/IPSFDefinition.cs new file mode 100644 index 000000000..263f421c3 --- /dev/null +++ b/cs/src/core/Index/PSF/IPSFDefinition.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; + +namespace FASTER.core +{ + /// + /// The definition of a single PSF (Predicate Subset Function) + /// + /// The data from the provider that the PSF should be run over + /// The data from the provider that the PSF should be run over + public interface IPSFDefinition + where TPSFKey : struct + { + /// + /// The callback used to obtain a new TPFSKey for the ProviderData record for this PSF definition. + /// + /// The representation of the data that was written to the primary store + /// (e.g. Upsert in FasterKV). + /// Null if the record does not match the PSF, else the indexing key for the record + public TPSFKey? Execute(TProviderData record); + + /// + /// The Name of the PSF, assigned by the caller. Must be unique among PSFs in the group. It is + /// used by the caller to index PSFs in the group in a friendly way. + /// + public string Name { get; } + + } +} diff --git a/cs/src/core/Index/PSF/IQueryPSF.cs b/cs/src/core/Index/PSF/IQueryPSF.cs new file mode 100644 index 000000000..1ec231af9 --- /dev/null +++ b/cs/src/core/Index/PSF/IQueryPSF.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Collections.Generic; + +namespace FASTER.core +{ + /// + /// Provides an interface on the that decouples the + /// PSFGroup from the primary FasterKV's TKVKey and TKVValue. + /// + /// + /// + public interface IQueryPSF + { + /// + /// Issues a query on the specified to return s. + /// + /// The ordinal of the in this group + /// The key to query on to rertrieve the s. + /// + IEnumerable Query(int psfOrdinal, TPSFKey key); + } +} diff --git a/cs/src/core/Index/PSF/PSF.cs b/cs/src/core/Index/PSF/PSF.cs new file mode 100644 index 000000000..8f5fcce8e --- /dev/null +++ b/cs/src/core/Index/PSF/PSF.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Collections.Generic; + +namespace FASTER.core +{ + /// + /// The implementation of the Predicate Subset Function. + /// + /// The type of the key returned by the Predicate and store in the secondary + /// FasterKV instance + /// The type of data record supplied by the data provider; in FasterKV it + /// is the logicalAddress of the record in the primary FasterKV instance. + public class PSF : IPSF + { + private readonly IQueryPSF psfGroup; + + internal int GroupOrdinal { get; } // in the psfGroup list + + internal int PsfOrdinal { get; } // in the psfGroup + + /// + public string Name { get; } + + internal PSF(int groupOrdinal, int psfOrdinal, string name, IQueryPSF iqp) + { + this.GroupOrdinal = groupOrdinal; + this.PsfOrdinal = psfOrdinal; + this.Name = name; + this.psfGroup = iqp; + } + + /// + /// Issues a query on this PSF to return s. + /// + /// + public IEnumerable Query(TPSFKey key) + => this.psfGroup.Query(this.PsfOrdinal, key); + } +} diff --git a/cs/src/core/Index/PSF/PSFCompositeKey.cs b/cs/src/core/Index/PSF/PSFCompositeKey.cs new file mode 100644 index 000000000..dc41e9078 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFCompositeKey.cs @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + /// + /// Wraps the set of TPSFKeys for a record in the secondary FasterKV instance. + /// + /// + public unsafe struct PSFCompositeKey + where TPSFKey : struct + { + // This class is entirely a "reinterpret_cast" implementation; there are no data members. + internal void CopyTo(ref PSFCompositeKey other, int keySize, int chainHeight) + { + var thisKeysPointer = (byte*)Unsafe.AsPointer(ref this); + var otherKeysPointer = (byte*)Unsafe.AsPointer(ref other); + var len = keySize * chainHeight; + Buffer.MemoryCopy(thisKeysPointer, otherKeysPointer, len, len); + } + + /// + /// Get a reference to the key for the PSF identified by psfOrdinal. + /// + /// The ordinal of the PSF in its parent PSFGroup + /// Size of the PSFKey struct + /// A reference to the key for the PSF identified by psfOrdinal. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ref TPSFKey GetKeyRef(int psfOrdinal, int keySize) + => ref Unsafe.AsRef((byte*)Unsafe.AsPointer(ref this) + keySize * psfOrdinal); + + /// + /// Equality comparison + /// + /// The key comparer to use + /// The ordinal of the PSF in its parent PSFGroup + /// Size of the PSFKey struct + /// The key to compare to + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool Equals(IFasterEqualityComparer comparer, int psfOrdinal, int keySize, ref TPSFKey otherKey) + => comparer.Equals(ref GetKeyRef(psfOrdinal, keySize), ref otherKey); + + internal class VarLenLength : IVariableLengthStruct> + { + private readonly int size; + + internal VarLenLength(int keySize, int chainHeight) => this.size = keySize * chainHeight; + + public int GetAverageLength() => this.size; + + public int GetInitialLength(ref Input _) => this.size; + + public int GetLength(ref PSFCompositeKey _) => this.size; + } + + internal class UnusedKeyComparer : IFasterEqualityComparer> + { + public long GetHashCode64(ref PSFCompositeKey cKey) + => throw new InvalidOperationException("Must use the overload with PSF ordinal"); + + public bool Equals(ref PSFCompositeKey cKey1, ref PSFCompositeKey cKey2) + => throw new InvalidOperationException("Must use the overload with PSF ordinal"); + } + internal unsafe struct PtrWrapper : IDisposable + { + private readonly SectorAlignedMemory mem; + private readonly int size; + + internal PtrWrapper(int size, SectorAlignedBufferPool pool) + { + this.size = size; + mem = pool.Get(size); + } + + internal void Set(ref TPSFKey key) + => Buffer.MemoryCopy(Unsafe.AsPointer(ref key), mem.GetValidPointer(), this.size, this.size); + + internal ref PSFCompositeKey GetRef() + => ref Unsafe.AsRef>(mem.GetValidPointer()); + + public void Dispose() => mem.Return(); + } + } + + internal interface ICompositeKeyComparer + { + int ChainHeight { get; } + + long GetHashCode64(int psfOrdinal, ref TCompositeKey cKey); + + bool Equals(bool isQuery, int psfOrdinal, ref TCompositeKey queryKey, ref TCompositeKey storedKey); + } + + internal class PSFCompositeKeyComparer : ICompositeKeyComparer> + where TPSFKey : struct + { + private readonly IFasterEqualityComparer userComparer; + private readonly int keySize; + public int ChainHeight { get; } + + internal PSFCompositeKeyComparer(IFasterEqualityComparer ucmp, int ksize, int chainHeight) + { + this.userComparer = ucmp; + this.keySize = ksize; + this.ChainHeight = chainHeight; + } + + public long GetHashCode64(int psfOrdinal, ref PSFCompositeKey cKey) + => this.userComparer.GetHashCode64(ref cKey.GetKeyRef(psfOrdinal, this.keySize)); + + public bool Equals(bool isQuery, int psfOrdinal, ref PSFCompositeKey queryKey, ref PSFCompositeKey storedKey) + => userComparer.Equals( + // For a query, the composite key consists of only one key, the query key, at ordinal 0. + ref queryKey.GetKeyRef(isQuery ? 0 : psfOrdinal, this.keySize), + ref storedKey.GetKeyRef(psfOrdinal, this.keySize)); + } +} diff --git a/cs/src/core/Index/PSF/PSFContext.cs b/cs/src/core/Index/PSF/PSFContext.cs new file mode 100644 index 000000000..cb4eae3a2 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFContext.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// Context for operations on the secondary FasterKV instance. + /// + public struct PSFContext + { + } +} diff --git a/cs/src/core/Index/PSF/PSFFunctions.cs b/cs/src/core/Index/PSF/PSFFunctions.cs new file mode 100644 index 000000000..5c16e9aa7 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFFunctions.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +using System; + +namespace FASTER.core +{ + /// + /// The Functions for the PSFValue-wrapped Value; mostly pass-through + /// + /// The type of the result key + /// The type of the value + public class PSFFunctions : IFunctions, PSFValue, PSFInputSecondary, + PSFOutputSecondary, PSFContext> + where TPSFKey: struct + where TRecordId: struct + { + IChainPost> chainPost; + + // TODO: remove stuff that has been moved to PSFOutput, etc. + + internal PSFFunctions(IChainPost> chainPost) + => this.chainPost = chainPost; + + #region Upserts + public bool ConcurrentWriter(ref PSFCompositeKey _, ref PSFValue src, ref PSFValue dst) + { + src.CopyTo(ref dst, this.chainPost.RecordIdSize, this.chainPost.ChainHeight); + return true; + } + + public void SingleWriter(ref PSFCompositeKey _, ref PSFValue src, ref PSFValue dst) + => src.CopyTo(ref dst, this.chainPost.RecordIdSize, this.chainPost.ChainHeight); + + public void UpsertCompletionCallback(ref PSFCompositeKey _, ref PSFValue value, PSFContext ctx) + { /* TODO */ } + #endregion Upserts + + #region Reads + public void ConcurrentReader(ref PSFCompositeKey key, ref PSFInputSecondary input, ref PSFValue value, ref PSFOutputSecondary dst) + => this.SingleReader(ref key, ref input, ref value, ref dst); + + public unsafe void SingleReader(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue value, ref PSFOutputSecondary dst) + { + /* TODO is SingleReader still used? + dst.RecordId = value.RecordId; + dst.previousChainLinkLogicalAddress = *(this.chainPost.GetChainLinkPtrs(ref value) + input.PsfOrdinal); + */ + } + + public void ReadCompletionCallback(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFOutputSecondary output, PSFContext ctx, Status status) + { /* TODO */ } + #endregion Reads + + #region RMWs + public void CopyUpdater(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue oldValue, ref PSFValue newValue) + => newValue = oldValue; // TODO ensure no deepcopy needed + + public void InitialUpdater(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue value) + { /* TODO */ } + + public bool InPlaceUpdater(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue value) + { return true; /* TODO */ } + + public void RMWCompletionCallback(ref PSFCompositeKey _, ref PSFInputSecondary input, PSFContext ctx, Status status) + { /* TODO */ } + #endregion RMWs + + public void DeleteCompletionCallback(ref PSFCompositeKey _, PSFContext ctx) + { /* TODO */ } + + public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) + { /* TODO */ } + } +} diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs new file mode 100644 index 000000000..a33a4a9ae --- /dev/null +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + /// + /// A group of s. Ideally, most records in the group will either match all + /// PSFs or none, for efficient use of log space. + /// + /// The type of the wrapper for the provider's data (obtained from TRecordId) + /// The type of the key returned by the Predicate and stored in the secondary FasterKV instance + /// The type of data record supplied by the data provider; in FasterKV it + /// is the logicalAddress of the record in the primary FasterKV instance. + public class PSFGroup : IExecutePSF, + IQueryPSF + where TPSFKey : struct + where TRecordId : struct + { + internal FasterKV, PSFValue, PSFInputSecondary, + PSFOutputSecondary, PSFContext, PSFFunctions> fht; + private readonly PSFFunctions functions; + internal IPSFDefinition[] psfDefinitions; + private readonly int ordinal; // in the parent's PSFGroup list TODO needed? + private readonly PSFRegistrationSettings regSettings; + + private readonly CheckpointSettings checkpointSettings; + private readonly int keySize = Utility.GetSize(default(TPSFKey)); + private readonly int recordIdSize = (Utility.GetSize(default(TRecordId)) + sizeof(long) - 1) & ~(sizeof(long) - 1); + + internal ConcurrentStack, + PSFValue, PSFInputSecondary, + PSFOutputSecondary, PSFContext, PSFFunctions>> freeSessions + = new ConcurrentStack, + PSFValue, PSFInputSecondary, + PSFOutputSecondary, PSFContext, PSFFunctions>>(); + internal ConcurrentBag, + PSFValue, PSFInputSecondary, + PSFOutputSecondary, PSFContext, PSFFunctions>> allSessions + = new ConcurrentBag, + PSFValue, PSFInputSecondary, + PSFOutputSecondary, PSFContext, PSFFunctions>>(); + + /// + /// The list of s in this group + /// + public PSF[] PSFs { get; private set; } + + private int ChainHeight => this.PSFs.Length; + + private readonly IFasterEqualityComparer userKeyComparer; + private readonly ICompositeKeyComparer> compositeKeyComparer; + private readonly IChainPost> chainPost; + + private readonly SectorAlignedBufferPool bufferPool; + + /// + /// Constructor + /// + /// The ordinal of this PSFGroup in the 's + /// PSFGroup list. + /// PSF definitions + /// Optional registration settings + public PSFGroup(int ordinal, IPSFDefinition[] defs, PSFRegistrationSettings regSettings) + { + // TODO check for null defs regSettings etc; for PSFs defined on a FasterKV instance we create intelligent + // defaults in regSettings but other clients will have to specify at least hashTableSize, logSettings, etc. + this.psfDefinitions = defs; + this.ordinal = ordinal; + this.regSettings = regSettings; + this.userKeyComparer = GetUserKeyComparer(); + + this.PSFs = defs.Select((def, ord) => new PSF(this.ordinal, ord, def.Name, this)).ToArray(); + this.compositeKeyComparer = new PSFCompositeKeyComparer(this.userKeyComparer, this.keySize, this.ChainHeight); + + // TODO doc (or change) chainPost terminology + this.chainPost = new PSFValue.ChainPost(this.ChainHeight, this.keySize); + + this.checkpointSettings = regSettings?.CheckpointSettings; + this.functions = new PSFFunctions(chainPost); + this.fht = new FasterKV, PSFValue, PSFInputSecondary, + PSFOutputSecondary, PSFContext, PSFFunctions>( + regSettings.HashTableSize, new PSFFunctions(chainPost), + regSettings.LogSettings, this.checkpointSettings, null /*SerializerSettings*/, + new PSFCompositeKey.UnusedKeyComparer(), + new VariableLengthStructSettings, PSFValue> + { + keyLength = new PSFCompositeKey.VarLenLength(this.keySize, this.ChainHeight), + valueLength = new PSFValue.VarLenLength(this.recordIdSize, this.ChainHeight) + } + ) + { chainPost = chainPost }; + + this.bufferPool = this.fht.hlog.bufferPool; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private ClientSession, PSFValue, PSFInputSecondary, + PSFOutputSecondary, PSFContext, + PSFFunctions> GetSession () + { + // Sessions are used only on post-RegisterPSF actions (Upsert, RMW, Query). + if (this.freeSessions.TryPop(out var session)) + return session; + session = this.fht.NewSession(threadAffinitized:this.regSettings.ThreadAffinitized); + this.allSessions.Add(session); + return session; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ReleaseSession(ClientSession, PSFValue, PSFInputSecondary, + PSFOutputSecondary, PSFContext, + PSFFunctions> session) + => this.freeSessions.Push(session); + + private IFasterEqualityComparer GetUserKeyComparer() + { + if (!(this.regSettings.KeyComparer is null)) + return this.regSettings.KeyComparer; + if (typeof(IFasterEqualityComparer).IsAssignableFrom(typeof(TPSFKey))) + return new TPSFKey() as IFasterEqualityComparer; + + Console.WriteLine( + $"***WARNING*** Creating default FASTER key equality comparer based on potentially slow {nameof(EqualityComparer)}." + + $" To avoid this, provide a comparer in {nameof(PSFRegistrationSettings)}.{nameof(PSFRegistrationSettings.KeyComparer)}," + + $" or make {typeof(TPSFKey).Name} implement the interface {nameof(IFasterEqualityComparer)}"); + return FasterEqualityComparer.Default; + } + + /// + /// Returns the named from the PSFs list. + /// + /// The name of the ; unique among all groups + /// + public PSF this[string name] + => Array.Find(this.PSFs, psf => psf.Name.Equals(name, StringComparison.CurrentCultureIgnoreCase)) + ?? throw new InvalidOperationException("TODO PSF Exception classes: PSF not found"); + + /// + public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recordId, bool isInserted) + { + // Note: stackalloc is safe here because PendingContext will copy it to the bufferPool if needed. + var keyMemLen = ((keySize * this.ChainHeight + sizeof(int) - 1) & ~(sizeof(int) - 1)) / sizeof(int); + var keyMem = stackalloc int[keyMemLen]; + for (var ii = 0; ii < keyMemLen; ++ii) + keyMem[ii] = 0; + ref PSFCompositeKey compositeKey = ref Unsafe.AsRef>(keyMem); + + var valueMem = stackalloc byte[recordIdSize + sizeof(long) * this.ChainHeight]; + ref PSFValue psfValue = ref Unsafe.AsRef>(valueMem); + psfValue.RecordId = recordId; + + bool* nullIndicators = stackalloc bool[this.ChainHeight]; + var anyMatch = false; + long* chainLinkPtrs = this.fht.chainPost.GetChainLinkPtrs(ref psfValue); + for (int ii = 0; ii < this.psfDefinitions.Length; ++ii) + { + var key = this.psfDefinitions[ii].Execute(providerData); + *(nullIndicators + ii) = !key.HasValue; + *(chainLinkPtrs + ii) = Constants.kInvalidAddress; + if (key.HasValue) + { + ref TPSFKey keyPtr = ref Unsafe.AsRef(keyMem + keySize * ii); + keyPtr = key.Value; + anyMatch = true; + } + } + + if (!anyMatch) + return Status.OK; + + var input = new PSFInputSecondary(0, compositeKeyComparer, nullIndicators); + var session = this.GetSession(); + Status status; + try // TODOperf: Assess overhead of try/finally + { + status = isInserted + ? session.PsfInsert(ref compositeKey, ref psfValue, ref input, 1 /*TODO: lsn*/) + : throw new NotImplementedException("TODO updates"); + } + finally + { + this.ReleaseSession(session); + } + return status; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool Verify(TProviderData providerData, int psfOrdinal) + => !(this.psfDefinitions[psfOrdinal].Execute(providerData) is null); + + /// + public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key) + { + // Create a varlen CompositeKey with just one item. This is ONLY used as the query key to + // PSFCompositeKeyComparer. Iterator functions cannot contain unsafe code or have byref args, + // and bufferPool is needed here because the stack goes away as part of the iterator operation. + var keyPtr = new PSFCompositeKey.PtrWrapper(this.keySize, this.bufferPool); + keyPtr.Set(ref key); + + // These indirections are necessary to avoid issues with passing ref or unsafe to a enumerator function. + // TODOperf: PSFInput* and PSFOutput* are classes because we communicate through interfaces to avoid + // having to add additional generic args. Interfaces on structs incur boxing overhead (plus the complexity + // of mutable structs). But check the performance here; if necessary perhaps I can figure out a way to + // pass a struct with no TRecordId, TPSFKey, etc. and use an FHT-level interface to manage it. + var psfInput = new PSFInputSecondary(psfOrdinal, compositeKeyComparer); + return Query(keyPtr, psfInput); + } + + private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKeyPtr, + PSFInputSecondary input) + { + var readArgs = new PSFReadArgs, PSFValue>( + input, new PSFOutputSecondary(this.chainPost)); + + var session = this.GetSession(); + Status status; + try + { + status = session.PsfReadKey(ref queryKeyPtr.GetRef(), ref readArgs, 1 /*TODO lsn*/); + if (status != Status.OK) // TODO check other status + yield break; + + var secondaryOutput = readArgs.Output as IPSFSecondaryOutput; + yield return secondaryOutput.RecordId; + + do + { + readArgs.Input.ReadLogicalAddress = secondaryOutput.PreviousLogicalAddress; + status = session.PsfReadAddress(ref readArgs, 1 /*TODO lsn*/); + if (status != Status.OK) // TODO check other status + yield break; + yield return secondaryOutput.RecordId; + } while (secondaryOutput.PreviousLogicalAddress != Constants.kInvalidAddress); + } + finally + { + this.ReleaseSession(session); + queryKeyPtr.Dispose(); + } + + } + } +} diff --git a/cs/src/core/Index/PSF/PSFInputSecondary.cs b/cs/src/core/Index/PSF/PSFInputSecondary.cs new file mode 100644 index 000000000..ad8700a4b --- /dev/null +++ b/cs/src/core/Index/PSF/PSFInputSecondary.cs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + /// + /// The additional input interface passed to the PSF functions for internal Insert, Read, etc. operations. + /// + /// The type of the Key, either a for the + /// secondary FasterKV instances, or the user's TKVKey for the primary FasterKV instance. + public interface IPSFInput + { + /// + /// The ordinal of the being queried; writable only for Insert. + /// + int PsfOrdinal { get; set; } + + /// + /// For Insert(), determine if the result of the at + /// was null; if so the value did not match and should not be stored in the key's chain. + /// + bool IsNullAt { get; } + + /// + /// For tracing back the chain, this is the next logicalAddress to get. + /// + long ReadLogicalAddress { get; set; } + + /// + /// Get the hashcode of the key of the at + /// + long GetHashCode64At(ref TKey cKey); + + /// + /// Determine if the keys match for the s + /// at . For query, the queryKey is a composite consisting of only one key, and + /// its ordinal is always zero. + /// + /// The composite key whose value is being matched with the store. If the + /// operation is Query, this is a composite consisting of only one key, the query key + /// The composite key in storage being compared to + /// + bool EqualsAt(ref TKey queryKey, ref TKey storedKey); + } + + /// + /// Input to PsfInternalReadAddress on the primary (stores user values) FasterKV to retrieve the Key and Value + /// for a logicalAddress returned from the secondary FasterKV instances. This class is FasterKV-provider-specific. + /// + /// The type of the key for user values + public unsafe class PSFInputPrimaryReadAddress : IPSFInput + { + internal PSFInputPrimaryReadAddress(long readLA) + { + this.ReadLogicalAddress = readLA; + } + + /// + public int PsfOrdinal + { + get => Constants.kInvalidPsfOrdinal; + set => throw new InvalidOperationException("Not valid for reading from Primary Address"); + } + + public bool IsNullAt => throw new InvalidOperationException("Not valid for reading from Primary Address"); + + public long ReadLogicalAddress { get; set; } + + public long GetHashCode64At(ref TKey key) + => throw new InvalidOperationException("Not valid for reading from Primary Address"); + + public bool EqualsAt(ref TKey queryKey, ref TKey storedKey) + => throw new InvalidOperationException("Not valid for reading from Primary Address"); + } + + /// + /// Input to operations on the secondary FasterKV instance (stores PSF chains) for everything + /// except reading based on a LogicalAddress. + /// + public unsafe class PSFInputSecondary : IPSFInput> + where TPSFKey : struct + { + internal readonly ICompositeKeyComparer> comparer; + internal readonly bool* nullIndicators; + + internal PSFInputSecondary(int psfOrd, ICompositeKeyComparer> keyCmp, + bool* nullIndicators = null) + { + this.PsfOrdinal = psfOrd; + this.comparer = keyCmp; + this.nullIndicators = nullIndicators; + this.ReadLogicalAddress = Constants.kInvalidAddress; + } + + public int PsfOrdinal { get; set; } + + public bool IsNullAt => this.nullIndicators[this.PsfOrdinal]; + + public long ReadLogicalAddress { get; set; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetHashCode64At(ref PSFCompositeKey cKey) + => this.comparer.GetHashCode64(this.PsfOrdinal, ref cKey); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool EqualsAt(ref PSFCompositeKey queryKey, ref PSFCompositeKey storedKey) + => this.comparer.Equals(this.nullIndicators is null, this.PsfOrdinal, ref queryKey, ref storedKey); + } +} diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs new file mode 100644 index 000000000..5c885b857 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; + +namespace FASTER.core +{ + internal class PSFManager where TRecordId : struct, IComparable + { + readonly List> psfGroups = new List>(); + + internal bool HasPSFs => this.psfGroups.Count > 0; + + internal Status Upsert(TProviderData providerData, TRecordId recordId, bool isInserted) + { + // TODO: This is called from ContextUpsert or InternalCompleteRetryRequest, where it's still + // in the Context threading control for the primaryKV. I think it needs to move out of there. + foreach (var group in this.psfGroups) + { + var status = group.ExecuteAndStore(providerData, recordId, isInserted); + if (status != Status.OK) + { + // TODO handle errors + } + } + return Status.OK; + } + + internal string[][] GetRegisteredPSFs() => throw new NotImplementedException("TODO"); + + internal PSF RegisterPSF(IPSFDefinition def, + PSFRegistrationSettings registrationSettings) + where TPSFKey : struct + { + // TODO: Runtime check that TPSFKey is blittable + var group = new PSFGroup(this.psfGroups.Count, new[] { def }, registrationSettings); + this.psfGroups.Add(group); + return group[def.Name]; + } + + internal PSF[] RegisterPSF(IPSFDefinition[] defs, + PSFRegistrationSettings registrationSettings) + where TPSFKey : struct + { + // TODO: Runtime check that TPSFKey is blittable + var group = new PSFGroup(this.psfGroups.Count, defs, registrationSettings); + this.psfGroups.Add(group); + return group.PSFs; + } + + internal IEnumerable QueryPSF(IPSFCreateProviderData providerDataCreator, + PSF psf, + TPSFKey psfKey, PSFQuerySettings querySettings) + where TPSFKey : struct + { + // TODO make sure 'psf' is from one of our groups + // TODO if we delete groups, the GroupOrdinals in the other groups' PSFs must be updated; change to IQueryPSF + var group = this.psfGroups[psf.GroupOrdinal]; + foreach (var recordId in psf.Query(psfKey)) + { + if (querySettings != null && querySettings.CancellationToken.IsCancellationRequested) + { + if (querySettings.ThrowOnCancellation) + querySettings.CancellationToken.ThrowIfCancellationRequested(); + yield break; + } + var providerData = providerDataCreator.Create(recordId); + if (providerData is null) + continue; + if (group.Verify(providerData, psf.PsfOrdinal)) + yield return providerData; + } + } + + internal IEnumerable QueryPSF(IPSFCreateProviderData providerDataCreator, + PSF psf, + TPSFKey[] psfKeys, PSFQuerySettings querySettings) + where TPSFKey : struct + { + // TODO make sure 'psf' is from one of our groups + + // TODO implement range queries. This will start retrieval of a stream of returned values for the + // chains for all specified keys for the PSF, returning them via an IEnumerable over a PQ + // (PriorityQueue) that is populated from each key's stream. This is how multiple range query bins + // are handled; the semantics are that a Union (via stream merge) of all records for all keys in the + // array is done. Obviously there will be a tradeoff between the granularity of the bins and the + // overhead of the PQ for the streams returned. + return QueryPSF(providerDataCreator, psf, psfKeys[0], querySettings); // TODO just to make the compiler happy + } + + internal IEnumerable QueryPSF( + IPSFCreateProviderData providerDataCreator, + PSF psf1, TPSFKey1 psfKey1, + PSF psf2, TPSFKey2 psfKey2, + Func matchPredicate, + PSFQuerySettings querySettings) + { + // TODO: full implementation via PQ + using var e1 = psf1.Query(psfKey1).GetEnumerator(); + using var e2 = psf2.Query(psfKey2).GetEnumerator(); + + var e1done = !e1.MoveNext(); + var e2done = !e2.MoveNext(); + + var group1 = this.psfGroups[psf1.GroupOrdinal]; + var group2 = this.psfGroups[psf2.GroupOrdinal]; + + var cancelToken = querySettings is null ? default : querySettings.CancellationToken; + bool cancellationRequested() + { + if (querySettings is null) + return false; + if (querySettings.ThrowOnCancellation) + cancelToken.ThrowIfCancellationRequested(); + return cancelToken.IsCancellationRequested; + } + + while (!e1done && !e2done) + { + if (cancellationRequested()) + yield break; + + // Descending order by recordId. TODO doc: Require IComparable on TRecordId + var cmp = e1.Current.CompareTo(e2.Current); + var predResult = cmp == 0 + ? matchPredicate(true, true) + : cmp > 0 ? matchPredicate(true, false) : matchPredicate(false, true); + if (predResult) + { + // Let the trailing one catch up + var providerData = providerDataCreator.Create(cmp < 0 ? e1.Current : e2.Current); + var verify = cmp <= 0 ? group1.Verify(providerData, psf1.PsfOrdinal) : true + && cmp >= 0 ? group2.Verify(providerData, psf2.PsfOrdinal) : true; + } + if (cmp <= 0) + e1done = !e1.MoveNext(); + if (cmp >= 0) + e2done = !e2.MoveNext(); + } + + // If all streams are done, normal conclusion. + if (e1done && e2done) + yield break; + + // At least one stream is still alive, but not all. See if they registered a callback. + if (!(querySettings is null) && !(querySettings.OnStreamEnded is null)) + { + if (!querySettings.OnStreamEnded(e1done ? (IPSF)psf1 : psf2, e1done ? 0 : 1)) + yield break; + } + + while ((!e1done || !e2done) && !cancellationRequested()) + { + var predResult = matchPredicate(!e1done, !e2done); + if (predResult) + { + //yield return providerDataCreator(!e1done ? e1.Current : e2.Current); + var providerData = providerDataCreator.Create(!e1done ? e1.Current : e2.Current); + if (!(providerData is null)) + { + var psfOrdinal = !e1done ? psf1.PsfOrdinal : psf2.PsfOrdinal; + if ((!e1done ? group1 : group2).Verify(providerData, psfOrdinal)) + yield return providerData; + } + } + if (!(!e1done ? e1 : e2).MoveNext()) + yield break; + } + } + + internal IEnumerable QueryPSF( + IPSFCreateProviderData providerDataCreator, + PSF psf1, TPSFKey1[] psfKeys1, + PSF psf2, TPSFKey2[] psfKeys2, + Func matchPredicate, + PSFQuerySettings querySettings) + { + // TODO: Similar range-query/PQ implementation (and first-element only execution) as discussed above. + return QueryPSF(providerDataCreator, psf1, psfKeys1[0], psf2, psfKeys2[0], matchPredicate, querySettings); + } + + // Power user versions. We could add up to 3. Anything more complicated than + // that, they can just post-process with LINQ. + + internal IEnumerable QueryPSF( + IPSFCreateProviderData providerDataCreator, + (PSF psf1, TPSFKey[])[] psfsAndKeys, + Func matchPredicate, + PSFQuerySettings querySettings) + { + // TODO: Not implemented. The input argument to the predicate is the matches to each + // element of psfsAndKeys. + return Array.Empty(); + } + + internal IEnumerable QueryPSF( + IPSFCreateProviderData providerDataCreator, + (PSF psf1, TPSFKey1[])[] psfsAndKeys1, + (PSF psf2, TPSFKey2[])[] psfsAndKeys2, + Func matchPredicate, + PSFQuerySettings querySettings) + { + // TODO: Not implemented. The first input argument to the predicate is the matches to each + // element of psfsAndKeys1; the second input argument to the predicate is the matches to each + // element of psfsAndKeys2. + return Array.Empty(); + } + } +} diff --git a/cs/src/core/Index/PSF/PSFOperationStatus.cs b/cs/src/core/Index/PSF/PSFOperationStatus.cs new file mode 100644 index 000000000..835cdb085 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFOperationStatus.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// Wrapper for the non-public OperationStatus + /// + public struct PSFOperationStatus + { + internal OperationStatus Status; + + internal PSFOperationStatus(OperationStatus opStatus) => this.Status = opStatus; + } +} diff --git a/cs/src/core/Index/PSF/PSFOutput.cs b/cs/src/core/Index/PSF/PSFOutput.cs new file mode 100644 index 000000000..6c9e1eef0 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFOutput.cs @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + /// + /// The additional output interface passed to the PSF functions for internal Insert, Read, etc. operations. + /// + /// The type of the Key, either a for the + /// secondary FasterKV instances, or the user's TKVKey for the primary FasterKV instance. + /// The type of the Key, either a for the + /// secondary FasterKV instances, or the user's TKVValue for the primary FasterKV instance. + public interface IPSFOutput + { + PSFOperationStatus Visit(int psfOrdinal, ref TKey key, ref TValue value, bool isConcurrent); + } + + public interface IPSFPrimaryOutput + { + TProviderData ProviderData { get; } + } + + /// + /// Output from ReadInternal on the primary (stores user values) FasterKV instance when reading + /// based on a LogicalAddress rather than a key. This class is FasterKV-provider-specific. + /// + /// The type of the key for user values + /// The type of the user values + public unsafe class PSFOutputPrimaryReadAddress : IPSFOutput, + IPSFPrimaryOutput> + where TKVKey : new() + where TKVValue : new() + { + private readonly AllocatorBase allocator; + + public FasterKVProviderData ProviderData { get; private set; } + + internal PSFOutputPrimaryReadAddress(AllocatorBase alloc) + { + this.allocator = alloc; + this.ProviderData = default; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus Visit(int psfOrdinal, ref TKVKey key, ref TKVValue value, bool isConcurrent) + { + this.ProviderData = new FasterKVProviderData(allocator, ref key, ref value); + return new PSFOperationStatus(OperationStatus.SUCCESS); + } + } + + public interface IPSFSecondaryOutput + { + TRecordId RecordId { get; } + long PreviousLogicalAddress { get; } + } + + /// + /// Output from operations on the secondary FasterKV instance (stores PSF chains). + /// + /// The type of the key returned from a + /// The type of the provider's record identifier + public unsafe class PSFOutputSecondary : IPSFOutput, PSFValue>, + IPSFSecondaryOutput + where TPSFKey : struct + where TRecordId : struct + { + IChainPost> chainPost; + + public TRecordId RecordId { get; private set; } + public long PreviousLogicalAddress { get; private set; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal PSFOutputSecondary(IChainPost> chainPost) + { + this.chainPost = chainPost; + this.RecordId = default; + this.PreviousLogicalAddress = Constants.kInvalidAddress; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus Visit(int psfOrdinal, ref PSFCompositeKey key, + ref PSFValue value, bool isConcurrent) + { + this.RecordId = value.RecordId; + this.PreviousLogicalAddress = *(this.chainPost.GetChainLinkPtrs(ref value) + psfOrdinal); + return new PSFOperationStatus(OperationStatus.SUCCESS); + } + } +} diff --git a/cs/src/core/Index/PSF/PSFQuerySession.cs b/cs/src/core/Index/PSF/PSFQuerySession.cs new file mode 100644 index 000000000..3b505473f --- /dev/null +++ b/cs/src/core/Index/PSF/PSFQuerySession.cs @@ -0,0 +1,285 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; + +namespace FASTER.core +{ + public sealed partial class ClientSession : + IPSFCreateProviderData>, + IDisposable + where Key : new() + where Value : new() + where Functions : IFunctions + { + #region PSF calls for Secondary FasterKV + internal Status PsfInsert(ref Key key, ref Value value, ref Input input, long serialNo) + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return fht.ContextPsfInsert(ref key, ref value, ref input, serialNo, ctx); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + internal Status PsfReadKey(ref Key key, ref PSFReadArgs psfArgs, long serialNo) + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return fht.ContextPsfReadKey(ref key, ref psfArgs, serialNo, ctx); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + internal Status PsfReadAddress(ref PSFReadArgs psfArgs, long serialNo) + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return fht.ContextPsfReadAddress(ref psfArgs, serialNo, ctx); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + #endregion PSF calls for Secondary FasterKV + + #region PSF Query API for primary FasterKV + /// + /// Issue a query on a single on a single key value. + /// + /// + /// foreach (var providerData in fht.QueryPSF(sizePsf, Size.Medium)) {...} + /// + /// The type of the key value to return results for + /// The Predicate Subset Function object + /// The key value to return results for + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + PSF psf, TPSFKey psfKey, PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return this.fht.PSFManager.QueryPSF(this, psf, psfKey, querySettings); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + /// + /// Issue a query on a single on multiple key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF(sizePsf, new TestPSFKey[] { Size.Medium, Size.Large })) {...} + /// + /// The type of the key value to return results for + /// The Predicate Subset Function object + /// A vector of key values to return results for; for example, an OR query on + /// a single PSF, or a range query for a PSF that generates keys identifying bins. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF + (PSF psf, TPSFKey[] psfKeys, PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return this.fht.PSFManager.QueryPSF(this, psf, psfKeys, querySettings); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + /// + /// Issue a query on a two s, each with a single key value. + /// + /// + /// var providerData in fht.QueryPSF(sizePsf, Size.Medium, colorPsf, Color.Red, (l, r) => l || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The first Predicate Subset Function object + /// The secojnd Predicate Subset Function object + /// The key value to return results from the first 's stored values + /// The key value to return results from the second 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + PSF psf1, TPSFKey1 psfKey1, + PSF psf2, TPSFKey2 psfKey2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return this.fht.PSFManager.QueryPSF(this, psf1, psfKey1, psf2, psfKey2, matchPredicate, querySettings); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + /// + /// Issue a query on a two s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }, + /// colorPsf, new TestPSFKey[] { Color.Red, Color.Blue}, + /// (l, r) => l || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The first Predicate Subset Function object + /// The secojnd Predicate Subset Function object + /// The key values to return results from the first 's stored values + /// The key values to return results from the second 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + PSF psf1, TPSFKey1[] psfKeys1, + PSF psf2, TPSFKey2[] psfKeys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return this.fht.PSFManager.QueryPSF(this, psf1, psfKeys1, psf2, psfKeys2, matchPredicate, querySettings); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + /// + /// Issue a query on one or more s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { + /// (sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), + /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue})}, + /// ll => ll[0])) + /// + /// The type of the key value for the vector + /// A vector of s and associated keys to be queried + /// A predicate that takes as a parameters a boolean vector in parallel with + /// the vector indicating whether a candidate record matches the corresponding + /// , and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of the input vector are true, + /// else false; an OR query would return true if element of the input vector is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + (PSF psf, TPSFKey[] keys)[] psfsAndKeys, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return this.fht.PSFManager.QueryPSF(this, psfsAndKeys, matchPredicate, querySettings); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + /// + /// Issue a query on multiple keys s for two different key types. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { + /// (sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), + /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue})}, + /// new[] {(sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), + /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue })}, + /// (ll, rr) => ll[0] || rr[0])) + /// + /// The type of the key value for the first vector's s + /// The type of the key value for the second vector's s + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A predicate that takes as a parameters a boolean vector in parallel with + /// the vector and a second boolean vector in parallel with + /// the vector, and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of both input vectors are true, + /// else false; an OR query would return true if any element of either input vector is true; and more complex + /// logic could be done depending on the specific PSFs. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + (PSF psf1, TPSFKey1[] keys1)[] psfsAndKeys1, + (PSF psf2, TPSFKey2[] keys2)[] psfsAndKeys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return this.fht.PSFManager.QueryPSF(this, psfsAndKeys1, psfsAndKeys2, matchPredicate, querySettings); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + #endregion PSF Query API for primary FasterKV + } +} diff --git a/cs/src/core/Index/PSF/PSFQuerySettings.cs b/cs/src/core/Index/PSF/PSFQuerySettings.cs new file mode 100644 index 000000000..c05e52fda --- /dev/null +++ b/cs/src/core/Index/PSF/PSFQuerySettings.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Threading; + +namespace FASTER.core +{ + /// + /// Defines settings that control some behaviors of the QueryPSF execution. + /// + public class PSFQuerySettings + { + /// One or more streams has ended. Input PSF whose stream ended, + /// and the index of that PSF in the parameters (left to right, or in the + /// case of PSF arrays as parameters, left top to bottom, then right top + /// to bottom. + /// + /// true to continue the enumeration, else false + public Func OnStreamEnded; + + /// Cancel the enumeration if set. Can be set by another thread, + /// e.g. one presenting results to a UI, or by StreamEnded. + /// + public CancellationToken CancellationToken { get; set; } + + /// When cancellation is reqested, simply terminate the enumeration + /// without throwing a CancellationException. + public bool ThrowOnCancellation { get; set; } + } +} diff --git a/cs/src/core/Index/PSF/PSFReadArgs.cs b/cs/src/core/Index/PSF/PSFReadArgs.cs new file mode 100644 index 000000000..2635183f3 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFReadArgs.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + internal struct PSFReadArgs + { + internal readonly IPSFInput Input; + internal readonly IPSFOutput Output; + + internal PSFReadArgs(IPSFInput input, IPSFOutput output) + { + this.Input = input; + this.Output = output; + } + } +} diff --git a/cs/src/core/Index/PSF/PSFRegistrationSettings.cs b/cs/src/core/Index/PSF/PSFRegistrationSettings.cs new file mode 100644 index 000000000..d2be31c54 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFRegistrationSettings.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// Options for PSF registration. + /// + public class PSFRegistrationSettings + { + /// + /// When registring new PSFs over an existing store, this is the logicalAddress in the primary + /// FasterKV at which indexing will be started. + /// + public long IndexFromAddress = Constants.kInvalidAddress; + + /// + /// The hash table size to be used in the PSF-implementing secondary FasterKV instances. + /// For PSFs defined on a FasterKV instance, if this is 0 or less, it will use the same value + /// passed to the primary FasterKV instance. + /// + public long HashTableSize = 0; + + /// + /// The log settings to be used in the PSF-implementing secondary FasterKV instances. + /// For PSFs defined on a FasterKV instance, if this is null, it will use the same settings as + /// passed to the primary FasterKV instance. + /// + public LogSettings LogSettings; + + /// + /// The log settings to be used in the PSF-implementing secondary FasterKV instances. + /// For PSFs defined on a FasterKV instance, if this is null, it will use the same settings + /// consistent with those passed to the primary FasterKV instance. + /// + public CheckpointSettings CheckpointSettings; + + /// + /// Optional key comparer; if null, should implement + /// ; otherwise a slower EqualityComparer will be used. + /// + public IFasterEqualityComparer KeyComparer; + + /// + /// Indicates whether PSFGroup Sessions are thread-affinitized. + /// + public bool ThreadAffinitized; + } +} diff --git a/cs/src/core/Index/PSF/PSFUpdateArgs.cs b/cs/src/core/Index/PSF/PSFUpdateArgs.cs new file mode 100644 index 000000000..b1cc3de2c --- /dev/null +++ b/cs/src/core/Index/PSF/PSFUpdateArgs.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + internal struct PSFUpdateArgs + { + internal long logicalAddress; // Set to the inserted or updated address + internal bool isInserted; // Set true if this was an insert rather than update or RMW + } +} diff --git a/cs/src/core/Index/PSF/PSFValue.cs b/cs/src/core/Index/PSF/PSFValue.cs new file mode 100644 index 000000000..efe7d9734 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFValue.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + /// + /// The TKVValue in the secondary, PSF-implementing FasterKV instances; it wraps the + /// and stores the links in the TPSFKey chains. + /// + /// The type of the provider's record identifier + public unsafe struct PSFValue + where TRecordId : struct + { + /// + /// LogicalAddress for FasterKV and FasterLog; something else for another data provider. + /// + public TRecordId RecordId; + + internal void CopyTo(ref PSFValue other, int recordIdSize, int chainHeight) + { + other.RecordId = this.RecordId; + var thisChainPointer = ((byte*)Unsafe.AsPointer(ref this) + recordIdSize); + var otherChainPointer = ((byte*)Unsafe.AsPointer(ref other) + recordIdSize); + var len = sizeof(long) * chainHeight; // The chains links are "long logicalAddress". + Buffer.MemoryCopy(thisChainPointer, otherChainPointer, len, len); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal long* GetChainLinkPtrs(int recordIdSize) + => (long*)((byte*)Unsafe.AsPointer(ref this) + recordIdSize); + + /// + public override string ToString() => $"RecordId = {this.RecordId}"; + + internal class VarLenLength : IVariableLengthStruct> + { + private readonly int size; + + internal VarLenLength(int recordIdSize, int chainHeight) => this.size = recordIdSize + sizeof(long) * chainHeight; + + public int GetAverageLength() => this.size; + + public int GetInitialLength(ref Input _) => this.size; + + public int GetLength(ref PSFValue _) => this.size; + } + + internal class ChainPost : IChainPost> + { + internal ChainPost(int chainHeight, int recordIdSize) + { + this.ChainHeight = chainHeight; + this.RecordIdSize = recordIdSize; + } + + public int ChainHeight { get; } + + public int RecordIdSize { get; } + + public long* GetChainLinkPtrs(ref PSFValue value) + => value.GetChainLinkPtrs(this.RecordIdSize); + } + } +} diff --git a/cs/test/RecoveryTests.cs b/cs/test/RecoveryTests.cs index b720f8b5a..db8c14f09 100644 --- a/cs/test/RecoveryTests.cs +++ b/cs/test/RecoveryTests.cs @@ -163,7 +163,6 @@ public void Populate(Action checkpointAction) using var session = fht.NewSession(); // Prpcess the batch of input data - bool first = true; for (int i = 0; i < numOps; i++) { session.RMW(ref inputArray[i].adId, ref inputArray[i], Empty.Default, i); From 751eefc05a7cce54a495a0fed3e6ccbcbd8c50f4 Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Thu, 4 Jun 2020 02:56:57 -0700 Subject: [PATCH 02/19] Implement PSF updates via RCU --- .../FasterPSFSample/BlittableOrders.cs | 51 +- cs/playground/FasterPSFSample/ColorKey.cs | 10 +- cs/playground/FasterPSFSample/Constants.cs | 6 +- cs/playground/FasterPSFSample/FPSF.cs | 4 +- .../FasterPSFSample/FasterPSFSample.cs | 216 +++++- cs/playground/FasterPSFSample/IOrders.cs | 6 +- cs/playground/FasterPSFSample/Key.cs | 28 +- cs/playground/FasterPSFSample/ObjectOrders.cs | 51 +- cs/playground/FasterPSFSample/ParseArgs.cs | 6 +- cs/playground/FasterPSFSample/SizeKey.cs | 10 +- cs/src/core/ClientSession/ClientSession.cs | 4 +- cs/src/core/Index/Common/Contexts.cs | 2 +- cs/src/core/Index/FASTER/FASTER.cs | 74 +- cs/src/core/Index/FASTER/FASTERBase.cs | 1 + cs/src/core/Index/FASTER/FASTERImpl.cs | 170 ++++- cs/src/core/Index/FASTER/FASTERThread.cs | 58 +- .../Index/PSF/FasterPSFContextOperations.cs | 126 ++++ cs/src/core/Index/PSF/FasterPSFImpl.cs | 661 ++---------------- ...ssion.cs => FasterPSFSessionOperations.cs} | 35 +- cs/src/core/Index/PSF/GroupKeys.cs | 67 ++ cs/src/core/Index/PSF/IChainPost.cs | 7 + cs/src/core/Index/PSF/IExecutePSF.cs | 31 +- cs/src/core/Index/PSF/PSF.cs | 6 +- cs/src/core/Index/PSF/PSFChangeTracker.cs | 88 +++ cs/src/core/Index/PSF/PSFCompositeKey.cs | 2 +- cs/src/core/Index/PSF/PSFExecutePhase.cs | 34 + cs/src/core/Index/PSF/PSFGroup.cs | 230 +++++- .../PSF/{PSFInputSecondary.cs => PSFInput.cs} | 53 +- cs/src/core/Index/PSF/PSFManager.cs | 78 ++- cs/src/core/Index/PSF/PSFOutput.cs | 16 +- .../core/Index/PSF/PSFRegistrationSettings.cs | 11 + cs/src/core/Index/PSF/PSFResultFlags.cs | 36 + cs/src/core/Index/PSF/PSFUpdateArgs.cs | 20 +- cs/src/core/Index/PSF/PSFValue.cs | 10 + 34 files changed, 1336 insertions(+), 872 deletions(-) create mode 100644 cs/src/core/Index/PSF/FasterPSFContextOperations.cs rename cs/src/core/Index/PSF/{PSFQuerySession.cs => FasterPSFSessionOperations.cs} (91%) create mode 100644 cs/src/core/Index/PSF/GroupKeys.cs create mode 100644 cs/src/core/Index/PSF/PSFChangeTracker.cs create mode 100644 cs/src/core/Index/PSF/PSFExecutePhase.cs rename cs/src/core/Index/PSF/{PSFInputSecondary.cs => PSFInput.cs} (70%) create mode 100644 cs/src/core/Index/PSF/PSFResultFlags.cs diff --git a/cs/playground/FasterPSFSample/BlittableOrders.cs b/cs/playground/FasterPSFSample/BlittableOrders.cs index 46994b969..0ac303faa 100644 --- a/cs/playground/FasterPSFSample/BlittableOrders.cs +++ b/cs/playground/FasterPSFSample/BlittableOrders.cs @@ -10,62 +10,71 @@ namespace FasterPSFSample public struct BlittableOrders : IOrders { // Colors, strings, and enums are not blittable so we use int - public int Size { get; set; } + public int SizeInt { get; set; } - public int Color { get; set; } + public int ColorArgb { get; set; } public int NumSold { get; set; } public BlittableOrders(Constants.Size size, Color color, int numSold) { - this.Size = (int)size; - this.Color = color.ToArgb(); + this.SizeInt = (int)size; + this.ColorArgb = color.ToArgb(); this.NumSold = numSold; } - public (int, int, int) MemberTuple => (this.Size, this.Color, this.NumSold); + public (int, int, int) MemberTuple => (this.SizeInt, this.ColorArgb, this.NumSold); - public override string ToString() => $"{(Constants.Size)this.Size}, {Constants.ColorDict[this.Color].Name}, {NumSold}"; + public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {NumSold}"; public class Functions : IFunctions, Context> { + #region Read public void ConcurrentReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) => dst.Value = value; + public void SingleReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) + => dst.Value = value; + + public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) + { + if (output.Value.MemberTuple != FasterPSFSample.keyDict[key].MemberTuple) + throw new Exception("Read mismatch error!"); + } + #endregion Read + + #region Upsert public bool ConcurrentWriter(ref Key key, ref BlittableOrders src, ref BlittableOrders dst) { dst = src; return true; } - public void CopyUpdater(ref Key key, ref Input input, ref BlittableOrders oldValue, ref BlittableOrders newValue) - => throw new NotImplementedException(); + public void SingleWriter(ref Key key, ref BlittableOrders src, ref BlittableOrders dst) + => dst = src; - public void InitialUpdater(ref Key key, ref Input input, ref BlittableOrders value) + public void UpsertCompletionCallback(ref Key key, ref BlittableOrders value, Context context) => throw new NotImplementedException(); + #endregion Upsert - public bool InPlaceUpdater(ref Key key, ref Input input, ref BlittableOrders value) + #region RMW + public void CopyUpdater(ref Key key, ref Input input, ref BlittableOrders oldValue, ref BlittableOrders newValue) => throw new NotImplementedException(); - public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) + public void InitialUpdater(ref Key key, ref Input input, ref BlittableOrders value) => throw new NotImplementedException(); - public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) + public bool InPlaceUpdater(ref Key key, ref Input input, ref BlittableOrders value) { - if (output.Value.MemberTuple != key.MemberTuple) - throw new Exception("Read mismatch error!"); + value.ColorArgb = FasterPSFSample.IPUColor; + return true; } public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) => throw new NotImplementedException(); + #endregion RMW - public void SingleReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) - => dst.Value = value; - - public void SingleWriter(ref Key key, ref BlittableOrders src, ref BlittableOrders dst) - => dst = src; - - public void UpsertCompletionCallback(ref Key key, ref BlittableOrders value, Context context) + public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) => throw new NotImplementedException(); public void DeleteCompletionCallback(ref Key key, Context context) diff --git a/cs/playground/FasterPSFSample/ColorKey.cs b/cs/playground/FasterPSFSample/ColorKey.cs index e8859ec7a..255adc3bb 100644 --- a/cs/playground/FasterPSFSample/ColorKey.cs +++ b/cs/playground/FasterPSFSample/ColorKey.cs @@ -8,14 +8,14 @@ namespace FasterPSFSample { public struct ColorKey : IFasterEqualityComparer { - public int Color; + public int ColorArgb; - public ColorKey(Color color) => this.Color = color.ToArgb(); + public ColorKey(Color color) => this.ColorArgb = color.ToArgb(); - public override string ToString() => Constants.ColorDict[this.Color].Name; + public override string ToString() => Constants.ColorDict[this.ColorArgb].Name; - public long GetHashCode64(ref ColorKey key) => Utility.GetHashCode(key.Color); + public long GetHashCode64(ref ColorKey key) => Utility.GetHashCode(key.ColorArgb); - public bool Equals(ref ColorKey k1, ref ColorKey k2) => k1.Color == k2.Color; + public bool Equals(ref ColorKey k1, ref ColorKey k2) => k1.ColorArgb == k2.ColorArgb; } } diff --git a/cs/playground/FasterPSFSample/Constants.cs b/cs/playground/FasterPSFSample/Constants.cs index d154abd98..80ad8d060 100644 --- a/cs/playground/FasterPSFSample/Constants.cs +++ b/cs/playground/FasterPSFSample/Constants.cs @@ -15,6 +15,7 @@ public enum Size Medium, Large, XLarge, + XXLarge, NumSizes } @@ -23,9 +24,10 @@ public enum Size [Color.Black.ToArgb()] = Color.Black, [Color.Red.ToArgb()] = Color.Red, [Color.Green.ToArgb()] = Color.Green, - [Color.Blue.ToArgb()] = Color.Blue + [Color.Blue.ToArgb()] = Color.Blue, + [Color.Purple.ToArgb()] = Color.Purple }; - static internal Color[] Colors = { Color.Black, Color.Red, Color.Green, Color.Blue }; + static internal Color[] Colors = { Color.Black, Color.Red, Color.Green, Color.Blue, Color.Purple }; } } diff --git a/cs/playground/FasterPSFSample/FPSF.cs b/cs/playground/FasterPSFSample/FPSF.cs index caa0ab2cb..77feb0b14 100644 --- a/cs/playground/FasterPSFSample/FPSF.cs +++ b/cs/playground/FasterPSFSample/FPSF.cs @@ -30,8 +30,8 @@ internal FPSF(bool useObjectValues, bool useReadCache) useObjectValues ? new SerializerSettings { valueSerializer = () => new TSerializer() } : null); - this.SizePsf = fht.RegisterPSF("sizePsf", (k, v) => new SizeKey((Constants.Size)v.Size)); - this.ColorPsf = fht.RegisterPSF("colorPsf", (k, v) => new ColorKey(Constants.ColorDict[v.Color])); + this.SizePsf = fht.RegisterPSF("sizePsf", (k, v) => new SizeKey((Constants.Size)v.SizeInt)); + this.ColorPsf = fht.RegisterPSF("colorPsf", (k, v) => new ColorKey(Constants.ColorDict[v.ColorArgb])); } internal void Close() diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs index d220162ad..fd645e7c6 100644 --- a/cs/playground/FasterPSFSample/FasterPSFSample.cs +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -3,13 +3,26 @@ using FASTER.core; using System; +using System.Collections.Generic; using System.Diagnostics; +using System.Drawing; +using System.Linq; +using System.Threading; namespace FasterPSFSample { public partial class FasterPSFSample { private const int UpsertCount = 100; + private static int blueCount; + private static int mediumCount; + private static int mediumBlueCount; + + internal static Dictionary keyDict = new Dictionary(); + + private static int nextId = 1000000000; + + internal static int IPUColor; static void Main(string[] argv) { @@ -34,7 +47,14 @@ internal static void RunSample() { RunUpserts(fpsf); RunReads(fpsf); - RunPSFs(fpsf); + var ok = QueryPSFs(fpsf) + && UpdateByUpsert(fpsf) + && UpdateByRMW(fpsf) + && Delete(fpsf); + Console.WriteLine("--------------------------------------------------------"); + Console.WriteLine(ok ? "Passed! All operations succeeded" + : "*** Failed! *** One or more operations did not succeed"); + Console.WriteLine(); } finally { @@ -54,18 +74,29 @@ internal static void RunUpserts(FPSF(); + using var session = fpsf.fht.NewSession(); + var context = new Context(); - for (int i = 0; i < UpsertCount; i++) + for (int i = 0; i < UpsertCount; i++) + { + // Leave the last value unassigned from each category (we'll use it to update later + var key = new Key(Interlocked.Increment(ref nextId) - 1); + var value = new TValue + { + SizeInt = rng.Next((int)Constants.Size.NumSizes - 1), + ColorArgb = Constants.Colors[rng.Next(Constants.Colors.Length - 1)].ToArgb(), + NumSold = rng.Next(OrdersBinKey.MaxOrders - 1) + }; + keyDict[key] = value; + if (value.ColorArgb == Color.Blue.ToArgb()) { - var key = new Key((Constants.Size)rng.Next((int)Constants.Size.NumSizes), - Constants.Colors[rng.Next((int)Constants.Colors.Length)], - rng.Next(OrdersBinKey.MaxOrders)); - var value = new TValue { Size = key.Size, Color = key.Color, NumSold = key.NumSold }; - session.Upsert(ref key, ref value, context, 0); + ++blueCount; + if (value.SizeInt == (int)Constants.Size.Medium) + ++mediumBlueCount; } + if (value.SizeInt == (int)Constants.Size.Medium) + ++mediumCount; + session.Upsert(ref key, ref value, context, 0); } Console.WriteLine($"Upserted {UpsertCount} elements"); @@ -86,45 +117,166 @@ internal static void RunReads(FPSF(); var readCount = UpsertCount * 2; - using (var session = fpsf.fht.NewSession()) - { - for (int i = 0; i < UpsertCount; i++) - { - var key = new Key((Constants.Size)rng.Next((int)Constants.Size.NumSizes), - Constants.Colors[rng.Next((int)Constants.Colors.Length)], - rng.Next(OrdersBinKey.MaxOrders)); - var status = session.Read(ref key, ref input, ref output, context, 0); + var keys = keyDict.Keys.ToArray(); - if (status == Status.OK && output.Value.MemberTuple != key.MemberTuple) - throw new Exception($"Error: Value does not match key in {nameof(RunReads)}"); - } + using var session = fpsf.fht.NewSession(); + for (int i = 0; i < UpsertCount; i++) + { + var key = keys[rng.Next(keys.Length)]; + var status = session.Read(ref key, ref input, ref output, context, 0); - session.CompletePending(true); + if (status == Status.OK && output.Value.MemberTuple != keyDict[key].MemberTuple) + throw new Exception($"Error: Value does not match key in {nameof(RunReads)}"); } + + session.CompletePending(true); Console.WriteLine($"Read {readCount} elements with {++statusPending} Pending"); } const string indent = " "; - internal static void RunPSFs(FPSF fpsf) + internal static bool QueryPSFs(FPSF fpsf) where TValue : IOrders, new() where TOutput : IOutput, new() where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { + Console.WriteLine(); Console.WriteLine("Querying PSFs from FASTER", UpsertCount); - using (var session = fpsf.fht.NewSession()) + using var session = fpsf.fht.NewSession(); + + var actualCount = 0; + var ok = true; + Console.WriteLine(); + Console.WriteLine("No join op; all Blues: "); + foreach (var providerData in session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Blue))) { - Console.Write("No join op; all Mediums: "); - foreach (var providerData in session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium))) - { - Console.WriteLine(); - ref TValue value = ref providerData.GetValue(); - Console.Write(indent + value); - } - Console.WriteLine(); + ++actualCount; + ref TValue value = ref providerData.GetValue(); + if (verbose) + Console.WriteLine(indent + value); + } + ok &= blueCount == actualCount; + Console.WriteLine(blueCount == actualCount + ? $"Blue Passed: expected == actual ({blueCount})" + : $"Blue Failed: expected ({blueCount}) != actual ({actualCount})"); + + actualCount = 0; + Console.WriteLine(); + Console.WriteLine("No join op; all Mediums: "); + foreach (var providerData in session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium))) + { + ++actualCount; + ref TValue value = ref providerData.GetValue(); + if (verbose) + Console.WriteLine(indent + value); } + ok &= mediumCount == actualCount; + Console.WriteLine(mediumCount == actualCount + ? $"Medium Passed: expected == actual ({mediumCount})" + : $"Medium Failed: expected ({mediumCount}) != actual ({actualCount})"); + return ok; + } + + internal static bool UpdateByUpsert(FPSF fpsf) + where TValue : IOrders, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Updating Sizes"); + + using var session = fpsf.fht.NewSession(); + var xxlDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.XXLarge)).ToArray(); + var mediumDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium)).ToArray(); + var expected = mediumDatas.Length; + Console.WriteLine(); + Console.Write($"Changing all Medium to XXLarge; initial counts Medium {mediumDatas.Length}, XXLarge {xxlDatas.Length}"); + + var context = new Context(); + foreach (var providerData in mediumDatas) + { + // Update the value + ref TValue value = ref providerData.GetValue(); + Debug.Assert(value.SizeInt == (int)Constants.Size.Medium); + value.SizeInt = (int)Constants.Size.XXLarge; + + // Reuse the same key + session.Upsert(ref providerData.GetKey(), ref value, context, 2); + } + Console.WriteLine(); + + xxlDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.XXLarge)).ToArray(); + mediumDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium)).ToArray(); + bool ok = xxlDatas.Length == expected && mediumDatas.Length == 0; + Console.Write(ok ? "Passed" : "*** Failed *** "); + Console.WriteLine($": Medium {mediumDatas.Length}, XXLarge {xxlDatas.Length}"); + return ok; + } + + internal static bool UpdateByRMW(FPSF fpsf) + where TValue : IOrders, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Updating Colors"); + + using var session = fpsf.fht.NewSession(); + var purpleDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Purple)).ToArray(); + var blueDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Blue)).ToArray(); + var expected = blueDatas.Length; + Console.WriteLine(); + Console.Write($"Changing all Blue to Purple; initial counts Blue {blueDatas.Length}, Purple {purpleDatas.Length}"); + + IPUColor = Color.Purple.ToArgb(); + var context = new Context(); + var input = default(Input); + foreach (var providerData in blueDatas) + { + // This will call Functions<>.InPlaceUpdater. + session.RMW(ref providerData.GetKey(), ref input, context, 3); + } + Console.WriteLine(); + + purpleDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Purple)).ToArray(); + blueDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Blue)).ToArray(); + bool ok = purpleDatas.Length == expected && blueDatas.Length == 0; + Console.Write(ok ? "Passed" : "*** Failed *** "); + Console.WriteLine($": Blue {blueDatas.Length}, Purple {purpleDatas.Length}"); + return ok; + } + + internal static bool Delete(FPSF fpsf) + where TValue : IOrders, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Deleting Colors"); + + using var session = fpsf.fht.NewSession(); + var redDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Red)).ToArray(); + Console.WriteLine(); + Console.Write($"Deleting all Reds; initial count {redDatas.Length}"); + + var context = new Context(); + foreach (var providerData in redDatas) + { + // This will call Functions<>.InPlaceUpdater. + session.Delete(ref providerData.GetKey(), context, 4); + } + Console.WriteLine(); + + redDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Red)).ToArray(); + var ok = redDatas.Length == 0; + Console.Write(ok ? "Passed" : "*** Failed *** "); + Console.WriteLine($": Red {redDatas.Length}"); + return ok; } } } diff --git a/cs/playground/FasterPSFSample/IOrders.cs b/cs/playground/FasterPSFSample/IOrders.cs index 364a722c3..cdae5b3ea 100644 --- a/cs/playground/FasterPSFSample/IOrders.cs +++ b/cs/playground/FasterPSFSample/IOrders.cs @@ -6,12 +6,12 @@ namespace FasterPSFSample public interface IOrders { // Colors, strings, and enums are not blittable so we use int - int Size { get; set; } + int SizeInt { get; set; } - int Color { get; set; } + int ColorArgb { get; set; } int NumSold { get; set; } - (int, int, int) MemberTuple => (this.Size, this.Color, this.NumSold); + (int, int, int) MemberTuple => (this.SizeInt, this.ColorArgb, this.NumSold); } } diff --git a/cs/playground/FasterPSFSample/Key.cs b/cs/playground/FasterPSFSample/Key.cs index 3f7e0c772..9b873ca95 100644 --- a/cs/playground/FasterPSFSample/Key.cs +++ b/cs/playground/FasterPSFSample/Key.cs @@ -2,33 +2,19 @@ // Licensed under the MIT license. using FASTER.core; -using System.Drawing; namespace FasterPSFSample { - public struct Key : IFasterEqualityComparer, IOrders + public struct Key : IFasterEqualityComparer { - public int Size { get; set; } + // Note: int instead of long because we won't use enough values to overflow and having it a + // different length than TRecordId (which is long) makes sure the PSFValue offsetting is in sync. + public int Id { get; set; } - public int Color { get; set; } + public Key(int id) => this.Id = id; - public int NumSold { get; set; } + public long GetHashCode64(ref Key key) => Utility.GetHashCode(key.Id); - public Key(Constants.Size size, Color color, int numSold) - { - this.Size = (int)size; - this.Color = color.ToArgb(); - this.NumSold = numSold; - } - - public (int, int, int) MemberTuple => (this.Size, this.Color, this.NumSold); - - public override string ToString() => $"{(Constants.Size)this.Size}, {Constants.ColorDict[this.Color].Name}, {NumSold}"; - - private long AsLong => (this.Size + this.Color) << 30 | this.NumSold; - - public long GetHashCode64(ref Key key) => Utility.GetHashCode(key.AsLong); - - public bool Equals(ref Key k1, ref Key k2) => k1.MemberTuple == k2.MemberTuple; + public bool Equals(ref Key k1, ref Key k2) => k1.Id == k2.Id; } } diff --git a/cs/playground/FasterPSFSample/ObjectOrders.cs b/cs/playground/FasterPSFSample/ObjectOrders.cs index 8b1352b36..af84a5ddd 100644 --- a/cs/playground/FasterPSFSample/ObjectOrders.cs +++ b/cs/playground/FasterPSFSample/ObjectOrders.cs @@ -9,9 +9,9 @@ namespace FasterPSFSample { public class ObjectOrders : IOrders { - public int Size { get => values[0]; set => values[0] = value; } + public int SizeInt { get => values[0]; set => values[0] = value; } - public int Color { get => values[1]; set => values[1] = value; } + public int ColorArgb { get => values[1]; set => values[1] = value; } public int NumSold { get => values[2]; set => values[2] = value; } @@ -21,14 +21,14 @@ public class ObjectOrders : IOrders public ObjectOrders(Constants.Size size, Color color, int numSold) { - this.Size = (int)size; - this.Color = color.ToArgb(); + this.SizeInt = (int)size; + this.ColorArgb = color.ToArgb(); this.NumSold = numSold; } - public (int, int, int) MemberTuple => (this.Size, this.Color, this.NumSold); + public (int, int, int) MemberTuple => (this.SizeInt, this.ColorArgb, this.NumSold); - public override string ToString() => $"{(Constants.Size)this.Size}, {Constants.ColorDict[this.Color].Name}, {NumSold}"; + public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {NumSold}"; public class Serializer : BinaryObjectSerializer { @@ -48,43 +48,52 @@ public override void Serialize(ref ObjectOrders obj) public class Functions : IFunctions, Context> { + #region Read public void ConcurrentReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) => dst.Value = value; + public void SingleReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) + => dst.Value = value; + + public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) + { + if (output.Value.MemberTuple != FasterPSFSample.keyDict[key].MemberTuple) + throw new Exception("Read mismatch error!"); + } + #endregion Read + + #region Upsert public bool ConcurrentWriter(ref Key key, ref ObjectOrders src, ref ObjectOrders dst) { dst = src; return true; } - public void CopyUpdater(ref Key key, ref Input input, ref ObjectOrders oldValue, ref ObjectOrders newValue) - => throw new NotImplementedException(); + public void SingleWriter(ref Key key, ref ObjectOrders src, ref ObjectOrders dst) + => dst = src; - public void InitialUpdater(ref Key key, ref Input input, ref ObjectOrders value) + public void UpsertCompletionCallback(ref Key key, ref ObjectOrders value, Context context) => throw new NotImplementedException(); + #endregion Upsert - public bool InPlaceUpdater(ref Key key, ref Input input, ref ObjectOrders value) + #region RMW + public void CopyUpdater(ref Key key, ref Input input, ref ObjectOrders oldValue, ref ObjectOrders newValue) => throw new NotImplementedException(); - public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) + public void InitialUpdater(ref Key key, ref Input input, ref ObjectOrders value) => throw new NotImplementedException(); - public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) + public bool InPlaceUpdater(ref Key key, ref Input input, ref ObjectOrders value) { - if (output.Value.MemberTuple != key.MemberTuple) - throw new Exception("Read mismatch error!"); + value.ColorArgb = FasterPSFSample.IPUColor; + return true; } public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) => throw new NotImplementedException(); + #endregion RMW - public void SingleReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) - => dst.Value = value; - - public void SingleWriter(ref Key key, ref ObjectOrders src, ref ObjectOrders dst) - => dst = src; - - public void UpsertCompletionCallback(ref Key key, ref ObjectOrders value, Context context) + public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) => throw new NotImplementedException(); public void DeleteCompletionCallback(ref Key key, Context context) diff --git a/cs/playground/FasterPSFSample/ParseArgs.cs b/cs/playground/FasterPSFSample/ParseArgs.cs index 7b7550015..36e015959 100644 --- a/cs/playground/FasterPSFSample/ParseArgs.cs +++ b/cs/playground/FasterPSFSample/ParseArgs.cs @@ -8,6 +8,7 @@ namespace FasterPSFSample public partial class FasterPSFSample { private static bool useObjectValue; + private static bool verbose; const string ObjValuesArg = "--objValues"; @@ -39,8 +40,11 @@ static bool Usage(string message = null) continue; } if (string.Compare(arg, "--help", ignoreCase: true) == 0 || arg == "/?" || arg == "-?") - { return Usage(); + if (string.Compare(arg, "-v", ignoreCase: true) == 0) + { + verbose = true; + continue; } return Usage($"Unknown argument: {arg}"); } diff --git a/cs/playground/FasterPSFSample/SizeKey.cs b/cs/playground/FasterPSFSample/SizeKey.cs index 484b3934c..1962248f9 100644 --- a/cs/playground/FasterPSFSample/SizeKey.cs +++ b/cs/playground/FasterPSFSample/SizeKey.cs @@ -7,14 +7,14 @@ namespace FasterPSFSample { public struct SizeKey : IFasterEqualityComparer { - public int Size; + public int SizeInt; - public SizeKey(Constants.Size size) => this.Size = (int)size; + public SizeKey(Constants.Size size) => this.SizeInt = (int)size; - public override string ToString() => ((Constants.Size)this.Size).ToString(); + public override string ToString() => ((Constants.Size)this.SizeInt).ToString(); - public long GetHashCode64(ref SizeKey key) => Utility.GetHashCode(key.Size); + public long GetHashCode64(ref SizeKey key) => Utility.GetHashCode(key.SizeInt); - public bool Equals(ref SizeKey k1, ref SizeKey k2) => k1.Size == k2.Size; + public bool Equals(ref SizeKey k1, ref SizeKey k2) => k1.SizeInt == k2.SizeInt; } } diff --git a/cs/src/core/ClientSession/ClientSession.cs b/cs/src/core/ClientSession/ClientSession.cs index 9069550ec..5dc5fd594 100644 --- a/cs/src/core/ClientSession/ClientSession.cs +++ b/cs/src/core/ClientSession/ClientSession.cs @@ -58,7 +58,7 @@ public FasterKVProviderData Create(long logicalAddress) var psfArgs = new PSFReadArgs(new PSFInputPrimaryReadAddress(logicalAddress), new PSFOutputPrimaryReadAddress(this.fht.hlog)); - // Call this directly here, because we are + // Call this directly here, because we are TODO var status = fht.ContextPsfReadAddress(ref psfArgs, 1 /*TODO lsn*/, ctx); var primaryOutput = psfArgs.Output as IPSFPrimaryOutput>; return status == Status.OK ? primaryOutput.ProviderData : null; // TODO check other states @@ -70,7 +70,7 @@ public FasterKVProviderData Create(long logicalAddress) public void Dispose() { CompletePending(true); - // TODO: this was removed; if intended, remove the method too: fht.DisposeClientSession(ID); + fht.DisposeClientSession(ID); // Session runs on a single thread if (!SupportAsync) diff --git a/cs/src/core/Index/Common/Contexts.cs b/cs/src/core/Index/Common/Contexts.cs index e83c36864..2c1519d88 100644 --- a/cs/src/core/Index/Common/Contexts.cs +++ b/cs/src/core/Index/Common/Contexts.cs @@ -89,7 +89,7 @@ internal struct PendingContext internal HashBucketEntry entry; internal LatchOperation heldLatch; internal PSFReadArgs psfReadArgs; - internal PSFUpdateArgs psfUpdateArgs; + internal PSFUpdateArgs psfUpdateArgs; public void Dispose() { diff --git a/cs/src/core/Index/FASTER/FASTER.cs b/cs/src/core/Index/FASTER/FASTER.cs index a29220842..af8cd1104 100644 --- a/cs/src/core/Index/FASTER/FASTER.cs +++ b/cs/src/core/Index/FASTER/FASTER.cs @@ -323,14 +323,16 @@ internal Status ContextUpsert(ref Key key, ref Value value, Context context, lon FasterExecutionContext sessionCtx) { var pcontext = default(PendingContext); - var updateArgs = new PSFUpdateArgs(); + var updateArgs = new PSFUpdateArgs(); var internalStatus = InternalUpsert(ref key, ref value, ref context, ref pcontext, sessionCtx, serialNo, ref updateArgs); Status status; if (internalStatus == OperationStatus.SUCCESS && this.PSFManager.HasPSFs) { - status = this.PSFManager.Upsert(new FasterKVProviderData(this.hlog, ref key, ref value), - updateArgs.logicalAddress, updateArgs.isInserted); + status = this.PSFManager.Upsert(updateArgs.ChangeTracker is null + ? new FasterKVProviderData(this.hlog, ref key, ref value) + : updateArgs.ChangeTracker.AfterData, + updateArgs.LogicalAddress, updateArgs.ChangeTracker); } else { @@ -348,15 +350,19 @@ internal Status ContextRMW(ref Key key, ref Input input, Context context, long s FasterExecutionContext sessionCtx) { var pcontext = default(PendingContext); - var internalStatus = InternalRMW(ref key, ref input, ref context, ref pcontext, sessionCtx, serialNo); + var updateArgs = new PSFUpdateArgs(); + var internalStatus = InternalRMW(ref key, ref input, ref context, ref pcontext, sessionCtx, + serialNo, ref updateArgs); Status status; - if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) + if (internalStatus == OperationStatus.SUCCESS && this.PSFManager.HasPSFs) { - status = (Status) internalStatus; + status = this.PSFManager.Update(updateArgs.ChangeTracker); } else { - status = HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); } sessionCtx.serialNum = serialNo; @@ -367,59 +373,21 @@ internal Status ContextRMW(ref Key key, ref Input input, Context context, long s internal Status ContextDelete(ref Key key, Context context, long serialNo, FasterExecutionContext sessionCtx) { var pcontext = default(PendingContext); - var internalStatus = InternalDelete(ref key, ref context, ref pcontext, sessionCtx, serialNo); + var updateArgs = new PSFUpdateArgs(); + + var internalStatus = InternalDelete(ref key, ref context, ref pcontext, sessionCtx, + serialNo, ref updateArgs); Status status; - if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) + if (internalStatus == OperationStatus.SUCCESS && this.PSFManager.HasPSFs) { - status = (Status) internalStatus; + status = this.PSFManager.Delete(updateArgs.ChangeTracker); } else { - status = HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); - } - - sessionCtx.serialNum = serialNo; - return status; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfInsert(ref Key key, ref Value value, ref Input input, long serialNo, - FasterExecutionContext sessionCtx) - { - var pcontext = default(PendingContext); - var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, - ref pcontext, sessionCtx, serialNo); - var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND - ? (Status)internalStatus - : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); - - sessionCtx.serialNum = serialNo; - return status; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfReadKey(ref Key key, ref PSFReadArgs psfArgs, long serialNo, - FasterExecutionContext sessionCtx) - { - var pcontext = default(PendingContext); - var internalStatus = this.PsfInternalReadKey(ref key, ref psfArgs, ref pcontext, sessionCtx, serialNo); - var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND - ? (Status)internalStatus - : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); - - sessionCtx.serialNum = serialNo; - return status; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfReadAddress(ref PSFReadArgs psfArgs, long serialNo, - FasterExecutionContext sessionCtx) - { - var pcontext = default(PendingContext); - var internalStatus = this.PsfInternalReadAddress(ref psfArgs, ref pcontext, sessionCtx, serialNo); - var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND ? (Status)internalStatus : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + } sessionCtx.serialNum = serialNo; return status; diff --git a/cs/src/core/Index/FASTER/FASTERBase.cs b/cs/src/core/Index/FASTER/FASTERBase.cs index 9090eb47c..83fff92b8 100644 --- a/cs/src/core/Index/FASTER/FASTERBase.cs +++ b/cs/src/core/Index/FASTER/FASTERBase.cs @@ -75,6 +75,7 @@ internal static class Constants public const long kTempInvalidAddress = 1; public const int kFirstValidAddress = 64; + public const long kInvalidPsfGroupId = -1; public const int kInvalidPsfOrdinal = -1; } diff --git a/cs/src/core/Index/FASTER/FASTERImpl.cs b/cs/src/core/Index/FASTER/FASTERImpl.cs index 55f730c6b..946c9cf0f 100644 --- a/cs/src/core/Index/FASTER/FASTERImpl.cs +++ b/cs/src/core/Index/FASTER/FASTERImpl.cs @@ -220,6 +220,23 @@ internal OperationStatus InternalRead( #region Upsert Operation + private FasterKVProviderData CreateProviderData(ref Key key, long physicalAddress) + => new FasterKVProviderData(this.hlog, ref key, ref hlog.GetValue(physicalAddress)); + + private void SetBeforeData(PSFChangeTracker, long> changeTracker, + ref Key key, long logicalAddress, long physicalAddress) + { + changeTracker.BeforeRecordId = logicalAddress; + changeTracker.BeforeData = CreateProviderData(ref key, physicalAddress); + } + + private void SetAfterData(PSFChangeTracker, long> changeTracker, + ref Key key, long logicalAddress, long physicalAddress) + { + changeTracker.AfterRecordId = logicalAddress; + changeTracker.AfterData = CreateProviderData(ref key, physicalAddress); + } + /// /// Upsert operation. Replaces the value corresponding to 'key' with provided 'value', if one exists /// else inserts a new record with 'key' and 'value'. @@ -230,7 +247,7 @@ internal OperationStatus InternalRead( /// Pending context used internally to store the context of the operation. /// Session context /// Operation serial number - /// For PSFs, returns the inserted or updated LogicalAddress + /// For PSFs, returns the inserted or updated LogicalAddress and provider data /// /// /// @@ -256,7 +273,7 @@ internal OperationStatus InternalUpsert( ref Key key, ref Value value, ref Context userContext, ref PendingContext pendingContext, FasterExecutionContext sessionCtx, - long lsn, ref PSFUpdateArgs psfArgs) + long lsn, ref PSFUpdateArgs psfArgs) { var status = default(OperationStatus); var bucket = default(HashBucket*); @@ -265,8 +282,7 @@ internal OperationStatus InternalUpsert( var physicalAddress = default(long); var latchOperation = default(LatchOperation); var latestRecordVersion = -1; - psfArgs.logicalAddress = Constants.kInvalidAddress; - psfArgs.isInserted = false; + psfArgs.LogicalAddress = Constants.kInvalidAddress; var hash = comparer.GetHashCode64(ref key); var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); @@ -297,16 +313,29 @@ internal OperationStatus InternalUpsert( out logicalAddress, out physicalAddress); } + + if (logicalAddress >= hlog.ReadOnlyAddress && this.PSFManager.HasPSFs) + { + // Save the PreUpdate values. + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.IPU; + } } #endregion - psfArgs.logicalAddress = logicalAddress; + psfArgs.LogicalAddress = logicalAddress; // Optimization for most common case if (sessionCtx.phase == Phase.REST && logicalAddress >= hlog.ReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) { if (functions.ConcurrentWriter(ref key, ref value, ref hlog.GetValue(physicalAddress))) { + if (this.PSFManager.HasPSFs) + { + SetAfterData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.IPU; + } return OperationStatus.SUCCESS; } } @@ -392,6 +421,11 @@ internal OperationStatus InternalUpsert( { if (functions.ConcurrentWriter(ref key, ref value, ref hlog.GetValue(physicalAddress))) { + if (this.PSFManager.HasPSFs) + { + SetAfterData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.IPU; + } status = OperationStatus.SUCCESS; goto LatchRelease; // Release shared latch (if acquired) } @@ -414,8 +448,34 @@ internal OperationStatus InternalUpsert( hlog.ShallowCopy(ref key, ref hlog.GetKey(newPhysicalAddress)); functions.SingleWriter(ref key, ref value, ref hlog.GetValue(newPhysicalAddress)); - psfArgs.logicalAddress = newLogicalAddress; - psfArgs.isInserted = true; + + if (this.PSFManager.HasPSFs) + { + psfArgs.LogicalAddress = newLogicalAddress; + var isInsert = logicalAddress < hlog.BeginAddress || + (logicalAddress >= hlog.HeadAddress && hlog.GetInfo(physicalAddress).Tombstone); + if (isInsert) + { + // Old logicalAddress is invalid (deos not exist) or was deleted, so this is an insert. + // This goes through the fast Insert path which does not create a changeTracker. + } + else if (logicalAddress >= hlog.HeadAddress) + { + // The old record was valid but not in mutable range (that's handled above), so this is an RCU + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + SetAfterData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.RCU; + } + else + { + // ah, old record slipped onto disk + hlog.GetInfo(newPhysicalAddress).Invalid = true; + psfArgs.ChangeTracker = null; + status = OperationStatus.RETRY_NOW; + goto LatchRelease; + } + } var updatedEntry = default(HashBucketEntry); updatedEntry.Tag = tag; @@ -453,6 +513,9 @@ internal OperationStatus InternalUpsert( pendingContext.logicalAddress = logicalAddress; pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; + + psfArgs.ChangeTracker = null; + pendingContext.psfUpdateArgs = psfArgs; } #endregion @@ -498,6 +561,7 @@ internal OperationStatus InternalUpsert( /// pending context created when the operation goes pending. /// Session context /// Operation serial number + /// For PSFs, returns the updated LogicalAddress and provider data /// /// /// @@ -526,7 +590,8 @@ internal OperationStatus InternalUpsert( internal OperationStatus InternalRMW( ref Key key, ref Input input, ref Context userContext, - ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) + ref PendingContext pendingContext, FasterExecutionContext sessionCtx, + long lsn, ref PSFUpdateArgs psfArgs) { var recordSize = default(int); var bucket = default(HashBucket*); @@ -567,6 +632,14 @@ internal OperationStatus InternalRMW( out logicalAddress, out physicalAddress); } + + if (logicalAddress >= hlog.ReadOnlyAddress && this.PSFManager.HasPSFs) + { + // Get the PreUpdate values (or the secondary FKV position in the IPUCache). + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.IPU; + } } #endregion @@ -575,6 +648,8 @@ internal OperationStatus InternalRMW( { if (functions.InPlaceUpdater(ref key, ref input, ref hlog.GetValue(physicalAddress))) { + if (!(psfArgs.ChangeTracker is null)) + SetAfterData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); return OperationStatus.SUCCESS; } } @@ -667,6 +742,8 @@ internal OperationStatus InternalRMW( if (functions.InPlaceUpdater(ref key, ref input, ref hlog.GetValue(physicalAddress))) { + if (!(psfArgs.ChangeTracker is null)) + SetAfterData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); status = OperationStatus.SUCCESS; goto LatchRelease; // Release shared latch (if acquired) } @@ -756,10 +833,32 @@ ref hlog.GetValue(physicalAddress), { // ah, old record slipped onto disk hlog.GetInfo(newPhysicalAddress).Invalid = true; + psfArgs.ChangeTracker = null; status = OperationStatus.RETRY_NOW; goto LatchRelease; } + if (this.PSFManager.HasPSFs) + { + psfArgs.LogicalAddress = newLogicalAddress; + var isInsert = logicalAddress < hlog.BeginAddress || hlog.GetInfo(physicalAddress).Tombstone; + if (isInsert) + { + // Old logicalAddress is invalid or deleted, so this is an Insert only. + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Insert; + } + else + { + // The old record was valid but not in mutable range (that's handled above), so this is an RCU + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + SetAfterData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.RCU; + } + } + var updatedEntry = default(HashBucketEntry); updatedEntry.Tag = tag; updatedEntry.Address = newLogicalAddress & Constants.kAddressMask; @@ -797,11 +896,14 @@ ref hlog.GetValue(physicalAddress), pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; pendingContext.heldLatch = heldOperation; + + psfArgs.ChangeTracker = null; + pendingContext.psfUpdateArgs = psfArgs; } - #endregion + #endregion - #region Latch release - LatchRelease: + #region Latch release + LatchRelease: { switch (latchOperation) { @@ -819,7 +921,7 @@ ref hlog.GetValue(physicalAddress), if (status == OperationStatus.RETRY_NOW) { - return InternalRMW(ref key, ref input, ref userContext, ref pendingContext, sessionCtx, lsn); + return InternalRMW(ref key, ref input, ref userContext, ref pendingContext, sessionCtx, lsn, ref psfArgs); } else { @@ -840,6 +942,7 @@ ref hlog.GetValue(physicalAddress), /// Pending context used internally to store the context of the operation. /// Session context /// Operation serial number + /// For PSFs, returns the updated LogicalAddress and provider data /// /// /// @@ -864,7 +967,8 @@ ref hlog.GetValue(physicalAddress), internal OperationStatus InternalDelete( ref Key key, ref Context userContext, - ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) + ref PendingContext pendingContext, FasterExecutionContext sessionCtx, + long lsn, ref PSFUpdateArgs psfArgs) { var status = default(OperationStatus); var bucket = default(HashBucket*); @@ -1013,6 +1117,14 @@ internal OperationStatus InternalDelete( functions.ConcurrentWriter(ref hlog.GetKey(physicalAddress), ref v, ref hlog.GetValue(physicalAddress)); } + if (this.PSFManager.HasPSFs) + { + // Get the PreUpdate values (or the secondary FKV position in the IPUCache). + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; + } + status = OperationStatus.SUCCESS; goto LatchRelease; // Release shared latch (if acquired) } @@ -1032,6 +1144,14 @@ internal OperationStatus InternalDelete( functions.ConcurrentWriter(ref hlog.GetKey(physicalAddress), ref v, ref hlog.GetValue(physicalAddress)); } + if (this.PSFManager.HasPSFs) + { + // Get the PreUpdate values (or the secondary FKV position in the IPUCache). + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; + } + status = OperationStatus.SUCCESS; goto LatchRelease; // Release shared latch (if acquired) } @@ -1067,6 +1187,14 @@ internal OperationStatus InternalDelete( if (foundEntry.word == entry.word) { + if (this.PSFManager.HasPSFs) + { + // Get the PreUpdate values (or the secondary FKV position in the IPUCache). + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; + } + status = OperationStatus.SUCCESS; goto LatchRelease; } @@ -1089,6 +1217,9 @@ internal OperationStatus InternalDelete( pendingContext.logicalAddress = logicalAddress; pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; + + psfArgs.ChangeTracker = null; + pendingContext.psfUpdateArgs = psfArgs; } #endregion @@ -1111,7 +1242,7 @@ internal OperationStatus InternalDelete( if (status == OperationStatus.RETRY_NOW) { - return InternalDelete(ref key, ref userContext, ref pendingContext, sessionCtx, lsn); + return InternalDelete(ref key, ref userContext, ref pendingContext, sessionCtx, lsn, ref psfArgs); } else { @@ -1227,6 +1358,7 @@ internal OperationStatus InternalContinuePendingRead( pendingContext.psfReadArgs.Output.Visit(pendingContext.psfReadArgs.Input.PsfOrdinal, ref pendingContext.key.Get(), ref hlog.GetContextRecordValue(ref request), + tombstone: false, // checked above isConcurrent: false); } else if (pendingContext.type == OperationType.PSF_READ_ADDRESS) @@ -1234,6 +1366,7 @@ ref hlog.GetContextRecordValue(ref request), var key = default(Key); // Reading by address does not have a key pendingContext.psfReadArgs.Output.Visit(pendingContext.psfReadArgs.Input.PsfOrdinal, ref key, ref hlog.GetContextRecordValue(ref request), + tombstone: false, // checked above isConcurrent: false); } @@ -1483,7 +1616,8 @@ ref hlog.GetContextRecordValue(ref request), #endregion Retry: - return InternalRMW(ref pendingContext.key.Get(), ref pendingContext.input, ref pendingContext.userContext, ref pendingContext, sessionCtx, pendingContext.serialNum); + return InternalRMW(ref pendingContext.key.Get(), ref pendingContext.input, ref pendingContext.userContext, + ref pendingContext, sessionCtx, pendingContext.serialNum, ref pendingContext.psfUpdateArgs); } #endregion @@ -1561,13 +1695,15 @@ ref pendingContext.value.Get(), case OperationType.DELETE: internalStatus = InternalDelete(ref pendingContext.key.Get(), ref pendingContext.userContext, - ref pendingContext, currentCtx, pendingContext.serialNum); + ref pendingContext, currentCtx, pendingContext.serialNum, + ref pendingContext.psfUpdateArgs); break; case OperationType.RMW: internalStatus = InternalRMW(ref pendingContext.key.Get(), ref pendingContext.input, ref pendingContext.userContext, - ref pendingContext, currentCtx, pendingContext.serialNum); + ref pendingContext, currentCtx, pendingContext.serialNum, + ref pendingContext.psfUpdateArgs); break; } diff --git a/cs/src/core/Index/FASTER/FASTERThread.cs b/cs/src/core/Index/FASTER/FASTERThread.cs index 8a78d6915..a0d8b8a41 100644 --- a/cs/src/core/Index/FASTER/FASTERThread.cs +++ b/cs/src/core/Index/FASTER/FASTERThread.cs @@ -212,33 +212,62 @@ internal void InternalCompleteRetryRequest(FasterExecutionContext opCtx, FasterE switch (pendingContext.type) { case OperationType.RMW: - internalStatus = InternalRMW(ref key, ref pendingContext.input, ref pendingContext.userContext, ref pendingContext, currentCtx, pendingContext.serialNum); + internalStatus = InternalRMW(ref key, ref pendingContext.input, ref pendingContext.userContext, + ref pendingContext, currentCtx, pendingContext.serialNum, + ref pendingContext.psfUpdateArgs); break; case OperationType.UPSERT: - internalStatus = InternalUpsert(ref key, ref value, ref pendingContext.userContext, ref pendingContext, currentCtx, pendingContext.serialNum, ref pendingContext.psfUpdateArgs); + internalStatus = InternalUpsert(ref key, ref value, ref pendingContext.userContext, ref pendingContext, + currentCtx, pendingContext.serialNum, ref pendingContext.psfUpdateArgs); + break; + case OperationType.PSF_INSERT: + internalStatus = PsfInternalInsert(ref key, ref value, ref pendingContext.input, + ref pendingContext, currentCtx, pendingContext.serialNum); break; case OperationType.DELETE: - internalStatus = InternalDelete(ref key, ref pendingContext.userContext, ref pendingContext, currentCtx, pendingContext.serialNum); + internalStatus = InternalDelete(ref key, ref pendingContext.userContext, ref pendingContext, currentCtx, + pendingContext.serialNum, ref pendingContext.psfUpdateArgs); break; case OperationType.READ: throw new FasterException("Cannot happen!"); } + var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(opCtx, currentCtx, pendingContext, internalStatus); - Status status; - if (internalStatus == OperationStatus.SUCCESS && this.PSFManager.HasPSFs) - { - status = this.PSFManager.Upsert(new FasterKVProviderData(this.hlog, ref key, ref value), - pendingContext.psfUpdateArgs.logicalAddress, pendingContext.psfUpdateArgs.isInserted); - } - else + if (status == Status.OK && this.PSFManager.HasPSFs) { - status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND - ? (Status)internalStatus - : status = HandleOperationStatus(opCtx, currentCtx, pendingContext, internalStatus); + switch (pendingContext.type) + { + case OperationType.UPSERT: + // Successful Upsert must have its PSFs executed and their keys stored. + status = this.PSFManager.Upsert(new FasterKVProviderData(this.hlog, ref key, ref value), + pendingContext.psfUpdateArgs.LogicalAddress, + pendingContext.psfUpdateArgs.ChangeTracker); + break; + case OperationType.PSF_INSERT: + var psfInput = (IPSFInput)pendingContext.input; + var updateOp = pendingContext.psfUpdateArgs.ChangeTracker.UpdateOp; + if (psfInput.IsDelete && (updateOp == UpdateOperation.IPU || updateOp == UpdateOperation.RCU)) + { + // RCU Insert of a tombstoned old record is followed by Insert of the new record. + if (pendingContext.psfUpdateArgs.ChangeTracker.FindGroup(psfInput.GroupId, out var ordinal)) + { + ref GroupKeysPair groupKeysPair = ref pendingContext.psfUpdateArgs.ChangeTracker.GetGroupRef(ordinal); + this.chainPost.SetRecordId(ref value, pendingContext.psfUpdateArgs.ChangeTracker.AfterRecordId); + var pcontext = default(PendingContext); + PsfRcuInsert(groupKeysPair.After, ref value, ref pendingContext.input, + ref pcontext, currentCtx, pendingContext.serialNum + 1); + } + } + break; + default: + break; + } } - // If done, callback user code. + // If done, callback user code. if (status == Status.OK || status == Status.NOTFOUND) { if (pendingContext.heldLatch == LatchOperation.Shared) @@ -263,7 +292,6 @@ internal void InternalCompleteRetryRequest(FasterExecutionContext opCtx, FasterE default: throw new FasterException("Operation type not allowed for retry"); } - } } #endregion diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs new file mode 100644 index 000000000..0da96cd3c --- /dev/null +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + public partial class FasterKV : FasterBase, + IFasterKV + where Key : new() + where Value : new() + where Functions : IFunctions + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfReadKey(ref Key key, ref PSFReadArgs psfArgs, long serialNo, + FasterExecutionContext sessionCtx) + { + var pcontext = default(PendingContext); + var internalStatus = this.PsfInternalReadKey(ref key, ref psfArgs, ref pcontext, sessionCtx, serialNo); + var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfReadAddress(ref PSFReadArgs psfArgs, long serialNo, + FasterExecutionContext sessionCtx) + { + var pcontext = default(PendingContext); + var internalStatus = this.PsfInternalReadAddress(ref psfArgs, ref pcontext, sessionCtx, serialNo); + var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfInsert(ref Key key, ref Value value, ref Input input, long serialNo, + FasterExecutionContext sessionCtx) + { + var pcontext = default(PendingContext); + var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, + ref pcontext, sessionCtx, serialNo); + var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfUpdate(ref GroupKeysPair groupKeysPair, ref Value value, ref Input input, long serialNo, + FasterExecutionContext sessionCtx, + PSFChangeTracker changeTracker) + where TRecordId : struct + { + var pcontext = default(PendingContext); + var psfInput = (IPSFInput)input; + + var groupKeys = groupKeysPair.Before; + unsafe { psfInput.SetFlags(groupKeys.ResultFlags); } + psfInput.IsDelete = true; + + var internalStatus = this.PsfInternalInsert(ref groupKeys.GetCompositeKeyRef(), ref value, ref input, + ref pcontext, sessionCtx, serialNo); + Status status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + + sessionCtx.serialNum = serialNo; + + if (status == Status.OK) + { + this.chainPost.SetRecordId(ref value, changeTracker.AfterRecordId); + return PsfRcuInsert(groupKeysPair.After, ref value, ref input, ref pcontext, sessionCtx, serialNo + 1); + } + return status; + } + + private Status PsfRcuInsert(GroupKeys groupKeys, ref Value value, ref Input input, + ref PendingContext pcontext, FasterExecutionContext sessionCtx, long serialNo) + { + var psfInput = (IPSFInput)input; + unsafe { psfInput.SetFlags(groupKeys.ResultFlags); } + psfInput.IsDelete = false; + var internalStatus = this.PsfInternalInsert(ref groupKeys.GetCompositeKeyRef(), ref value, ref input, + ref pcontext, sessionCtx, serialNo /* todo */); + return internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfDelete(ref Key key, ref Value value, ref Input input, long serialNo, + FasterExecutionContext sessionCtx, + PSFChangeTracker changeTracker) + where TRecordId : struct + { + var pcontext = default(PendingContext); + + // TODO check cache first: + // var internalStatus = this.PsfInternalDeleteByCache(ref key, ref input, + // ref pcontext, sessionCtx, serialNo)); + // if (internalStatus == OperationStatus.NOTFOUND) + // ... do below ... + + var psfInput = (IPSFInput)input; + psfInput.IsDelete = true; + var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, + ref pcontext, sessionCtx, serialNo); + Status status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } + } +} diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index deae742ce..b1e8ac705 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -55,8 +55,9 @@ internal OperationStatus PsfInternalReadKey( status = OperationStatus.CPR_SHIFT_DETECTED; goto CreatePendingContext; // Pivot thread } - return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), - ref readcache.GetValue(physicalAddress), isConcurrent: false).Status; + return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref readcache.GetValue(physicalAddress), + hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: false).Status; } if (logicalAddress >= hlog.HeadAddress) @@ -95,19 +96,17 @@ internal OperationStatus PsfInternalReadKey( // Mutable region (even fuzzy region is included here) if (logicalAddress >= hlog.SafeReadOnlyAddress) { - return hlog.GetInfo(physicalAddress).Tombstone - ? OperationStatus.NOTFOUND - : psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), - ref hlog.GetValue(physicalAddress), isConcurrent:true).Status; + return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref hlog.GetValue(physicalAddress), + hlog.GetInfo(physicalAddress).Tombstone, isConcurrent:true).Status; } // Immutable region else if (logicalAddress >= hlog.HeadAddress) { - return hlog.GetInfo(physicalAddress).Tombstone - ? OperationStatus.NOTFOUND - : psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), - ref hlog.GetValue(physicalAddress), isConcurrent:true).Status; + return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref hlog.GetValue(physicalAddress), + hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: true).Status; } // On-Disk Region @@ -185,7 +184,8 @@ internal OperationStatus PsfInternalReadAddress( goto CreatePendingContext; // Pivot thread } return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), - ref readcache.GetValue(physicalAddress), isConcurrent: false).Status; + ref readcache.GetValue(physicalAddress), + hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: false).Status; } if (logicalAddress >= hlog.HeadAddress) @@ -207,25 +207,40 @@ internal OperationStatus PsfInternalReadAddress( // Mutable region (even fuzzy region is included here) if (logicalAddress >= hlog.SafeReadOnlyAddress) { - return hlog.GetInfo(physicalAddress).Tombstone - ? OperationStatus.NOTFOUND - : psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), - ref hlog.GetValue(physicalAddress), isConcurrent: true).Status; + return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref hlog.GetValue(physicalAddress), + hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: true).Status; } // Immutable region else if (logicalAddress >= hlog.HeadAddress) { - return hlog.GetInfo(physicalAddress).Tombstone - ? OperationStatus.NOTFOUND - : psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), - ref hlog.GetValue(physicalAddress), isConcurrent: true).Status; + return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref hlog.GetValue(physicalAddress), + hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: true).Status; } // On-Disk Region else if (logicalAddress >= hlog.BeginAddress) { status = OperationStatus.RECORD_ON_DISK; +#if false // TODO: See discussion for this in Teams (and email) + if (sessionCtx.phase == Phase.PREPARE) + { + Debug.Assert(heldOperation != LatchOperation.Exclusive); + if (heldOperation == LatchOperation.Shared || HashBucket.TryAcquireSharedLatch(bucket)) + heldOperation = LatchOperation.Shared; + else + status = OperationStatus.CPR_SHIFT_DETECTED; + + if (RelaxedCPR) // don't hold on to shared latched during IO + { + if (heldOperation == LatchOperation.Shared) + HashBucket.ReleaseSharedLatch(bucket); + heldOperation = LatchOperation.None; + } + } +#endif goto CreatePendingContext; } else @@ -234,10 +249,10 @@ internal OperationStatus PsfInternalReadAddress( return OperationStatus.NOTFOUND; } - #endregion +#endregion - #region Create pending context - CreatePendingContext: +#region Create pending context + CreatePendingContext: { pendingContext.type = OperationType.PSF_READ_ADDRESS; pendingContext.key = default; @@ -251,7 +266,7 @@ internal OperationStatus PsfInternalReadAddress( pendingContext.heldLatch = LatchOperation.None; pendingContext.psfReadArgs = psfArgs; } - #endregion +#endregion return status; } @@ -272,19 +287,23 @@ internal OperationStatus PsfInternalInsert( var psfInput = input as IPSFInput; - // Update the PSFValue links for chains with nullIndicator false (indicating a match with the + // Update the PSFValue links for chains with IsNullAt false (indicating a match with the // corresponding PSF) to point to the previous records for all keys in the composite key. // TODO: We're not checking for a previous occurrence of the PSFValue's recordId because - // we are doing insert only here; the update part of upsert is done in PsfInternalRMW. + // we are doing insert only here; the update part of upsert is done in PsfInternalUpdate. var chainHeight = this.chainPost.ChainHeight; long* hashes = stackalloc long[chainHeight]; long* chainLinkPtrs = this.chainPost.GetChainLinkPtrs(ref value); for (var chainLinkIdx = 0; chainLinkIdx < chainHeight; ++chainLinkIdx) { + // For RCU, or in case we had to retry due to CPR_SHIFT and somehow managed to delete + // the previously found record, clear out the chain link pointer. + long* chainLinkPtr = chainLinkPtrs + chainLinkIdx; + *chainLinkPtr = Constants.kInvalidAddress; + psfInput.PsfOrdinal = chainLinkIdx; if (psfInput.IsNullAt) continue; - long* chainLinkPtr = chainLinkPtrs + chainLinkIdx; var hash = psfInput.GetHashCode64At(ref compositeKey); *(hashes + chainLinkIdx) = hash; @@ -293,15 +312,18 @@ internal OperationStatus PsfInternalInsert( if (sessionCtx.phase != Phase.REST) HeavyEnter(hash, sessionCtx); - #region Trace back for record in in-memory HybridLog +#region Trace back for record in in-memory HybridLog entry = default; var tagExists = FindTag(hash, tag, ref bucket, ref slot, ref entry); if (tagExists) { logicalAddress = entry.Address; - // TODO: is this needed? If so, verify handling for multiple keys (should abandon here and restart on retry) - if (UseReadCache && ReadFromCache(ref compositeKey, ref logicalAddress, ref physicalAddress, ref latestRecordVersion)) + // TODO: If this happens for any of the TPSFKeys in the composite key, we'll create the pending + // context and come back here on the retry and overwrite any previously-obtained logicalAddress + // at the chainLinkPtr. + if (UseReadCache && ReadFromCache(ref compositeKey, ref logicalAddress, ref physicalAddress, + ref latestRecordVersion, psfInput)) { if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) { @@ -316,7 +338,6 @@ internal OperationStatus PsfInternalInsert( if (latestRecordVersion == -1) latestRecordVersion = hlog.GetInfo(physicalAddress).Version; - if (!psfInput.EqualsAt(ref compositeKey, ref hlog.GetKey(physicalAddress))) { logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; @@ -329,13 +350,16 @@ internal OperationStatus PsfInternalInsert( } } - if (!hlog.GetInfo(physicalAddress).Tombstone) + // The chain might extend past the tombstoned record so we must include it in the chain. + // TODO: check to see if the tombstoned record's chain link for this psf is kInvalidAddress + // and disinclude it if so. + //if (!hlog.GetInfo(physicalAddress).Tombstone) *chainLinkPtr = logicalAddress; } - #endregion +#endregion } - #region Entry latch operation +#region Entry latch operation if (sessionCtx.phase != Phase.REST) { switch (sessionCtx.phase) @@ -405,11 +429,11 @@ internal OperationStatus PsfInternalInsert( break; } } - #endregion +#endregion Debug.Assert(latestRecordVersion <= sessionCtx.version); - #region Create new record in the mutable region +#region Create new record in the mutable region CreateNewRecord: { // Immutable region or new record @@ -417,7 +441,7 @@ internal OperationStatus PsfInternalInsert( BlockAllocate(recordSize, out long newLogicalAddress, sessionCtx); var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); RecordInfo.WriteInfo(ref hlog.GetInfo(newPhysicalAddress), sessionCtx.version, - final:true, tombstone:false, invalidBit:false, + final:true, tombstone: psfInput.IsDelete, invalidBit:false, Constants.kInvalidAddress); // We manage all prev addresses within PSFValue hlog.ShallowCopy(ref compositeKey, ref hlog.GetKey(newPhysicalAddress)); functions.SingleWriter(ref compositeKey, ref value, ref hlog.GetValue(newPhysicalAddress)); @@ -494,570 +518,5 @@ internal OperationStatus PsfInternalInsert( ? PsfInternalInsert(ref compositeKey, ref value, ref input, ref pendingContext, sessionCtx, lsn) : status; } - - internal OperationStatus PsfInternalRMW( // TODO - ref Key key, ref Input input, - ref Context userContext, - ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) - { - var recordSize = default(int); - var bucket = default(HashBucket*); - var slot = default(int); - var logicalAddress = Constants.kInvalidAddress; - var physicalAddress = default(long); - var latestRecordVersion = -1; - var status = default(OperationStatus); - var latchOperation = LatchOperation.None; - var heldOperation = LatchOperation.None; - - var hash = comparer.GetHashCode64(ref key); - var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); - - if (sessionCtx.phase != Phase.REST) - HeavyEnter(hash, sessionCtx); - - #region Trace back for record in in-memory HybridLog - var entry = default(HashBucketEntry); - FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry, hlog.BeginAddress); - logicalAddress = entry.Address; - - // For simplicity, we don't let RMW operations use read cache - if (UseReadCache) - SkipReadCache(ref logicalAddress, ref latestRecordVersion); - var latestLogicalAddress = logicalAddress; - - if (logicalAddress >= hlog.HeadAddress) - { - physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - latestRecordVersion = hlog.GetInfo(physicalAddress).Version; - - if (!comparer.Equals(ref key, ref hlog.GetKey(physicalAddress))) - { - logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; - TraceBackForKeyMatch(ref key, logicalAddress, - hlog.HeadAddress, - out logicalAddress, - out physicalAddress); - } - } - #endregion - - // Optimization for the most common case - if (sessionCtx.phase == Phase.REST && logicalAddress >= hlog.ReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) - { - if (functions.InPlaceUpdater(ref key, ref input, ref hlog.GetValue(physicalAddress))) - { - return OperationStatus.SUCCESS; - } - } - - #region Entry latch operation - if (sessionCtx.phase != Phase.REST) - { - switch (sessionCtx.phase) - { - case Phase.PREPARE: - { - Debug.Assert(pendingContext.heldLatch != LatchOperation.Exclusive); - if (pendingContext.heldLatch == LatchOperation.Shared || HashBucket.TryAcquireSharedLatch(bucket)) - { - // Set to release shared latch (default) - latchOperation = LatchOperation.Shared; - if (latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) - { - status = OperationStatus.CPR_SHIFT_DETECTED; - goto CreateFailureContext; // Pivot Thread - } - break; // Normal Processing - } - else - { - status = OperationStatus.CPR_SHIFT_DETECTED; - goto CreateFailureContext; // Pivot Thread - } - } - case Phase.IN_PROGRESS: - { - if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) - { - Debug.Assert(pendingContext.heldLatch != LatchOperation.Shared); - if (pendingContext.heldLatch == LatchOperation.Exclusive || HashBucket.TryAcquireExclusiveLatch(bucket)) - { - // Set to release exclusive latch (default) - latchOperation = LatchOperation.Exclusive; - goto CreateNewRecord; // Create a (v+1) record - } - else - { - status = OperationStatus.RETRY_LATER; - goto CreateFailureContext; // Go Pending - } - } - break; // Normal Processing - } - case Phase.WAIT_PENDING: - { - if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) - { - if (HashBucket.NoSharedLatches(bucket)) - { - goto CreateNewRecord; // Create a (v+1) record - } - else - { - status = OperationStatus.RETRY_LATER; - goto CreateFailureContext; // Go Pending - } - } - break; // Normal Processing - } - case Phase.WAIT_FLUSH: - { - if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) - { - goto CreateNewRecord; // Create a (v+1) record - } - break; // Normal Processing - } - default: - break; - } - } - #endregion - - Debug.Assert(latestRecordVersion <= sessionCtx.version); - - #region Normal processing - - // Mutable Region: Update the record in-place - if (logicalAddress >= hlog.ReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) - { - if (FoldOverSnapshot) - { - Debug.Assert(hlog.GetInfo(physicalAddress).Version == sessionCtx.version); - } - - if (functions.InPlaceUpdater(ref key, ref input, ref hlog.GetValue(physicalAddress))) - { - status = OperationStatus.SUCCESS; - goto LatchRelease; // Release shared latch (if acquired) - } - } - - // Fuzzy Region: Must go pending due to lost-update anomaly - else if (logicalAddress >= hlog.SafeReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) - { - status = OperationStatus.RETRY_LATER; - // Do not retain latch for pendings ops in relaxed CPR - if (!RelaxedCPR) - { - // Retain the shared latch (if acquired) - if (latchOperation == LatchOperation.Shared) - { - heldOperation = latchOperation; - latchOperation = LatchOperation.None; - } - } - goto CreateFailureContext; // Go pending - } - - // Safe Read-Only Region: Create a record in the mutable region - else if (logicalAddress >= hlog.HeadAddress) - { - goto CreateNewRecord; - } - - // Disk Region: Need to issue async io requests - else if (logicalAddress >= hlog.BeginAddress) - { - status = OperationStatus.RECORD_ON_DISK; - // Do not retain latch for pendings ops in relaxed CPR - if (!RelaxedCPR) - { - // Retain the shared latch (if acquired) - if (latchOperation == LatchOperation.Shared) - { - heldOperation = latchOperation; - latchOperation = LatchOperation.None; - } - } - goto CreateFailureContext; // Go pending - } - - // No record exists - create new - else - { - goto CreateNewRecord; - } - - #endregion - - #region Create new record - CreateNewRecord: - { - recordSize = (logicalAddress < hlog.BeginAddress) ? - hlog.GetInitialRecordSize(ref key, ref input) : - hlog.GetRecordSize(physicalAddress); - BlockAllocate(recordSize, out long newLogicalAddress, sessionCtx); - var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); - RecordInfo.WriteInfo(ref hlog.GetInfo(newPhysicalAddress), sessionCtx.version, - true, false, false, - latestLogicalAddress); - hlog.ShallowCopy(ref key, ref hlog.GetKey(newPhysicalAddress)); - if (logicalAddress < hlog.BeginAddress) - { - functions.InitialUpdater(ref key, ref input, ref hlog.GetValue(newPhysicalAddress)); - status = OperationStatus.NOTFOUND; - } - else if (logicalAddress >= hlog.HeadAddress) - { - if (hlog.GetInfo(physicalAddress).Tombstone) - { - functions.InitialUpdater(ref key, ref input, ref hlog.GetValue(newPhysicalAddress)); - status = OperationStatus.NOTFOUND; - } - else - { - functions.CopyUpdater(ref key, ref input, - ref hlog.GetValue(physicalAddress), - ref hlog.GetValue(newPhysicalAddress)); - status = OperationStatus.SUCCESS; - } - } - else - { - // ah, old record slipped onto disk - hlog.GetInfo(newPhysicalAddress).Invalid = true; - status = OperationStatus.RETRY_NOW; - goto LatchRelease; - } - - var updatedEntry = default(HashBucketEntry); - updatedEntry.Tag = tag; - updatedEntry.Address = newLogicalAddress & Constants.kAddressMask; - updatedEntry.Pending = entry.Pending; - updatedEntry.Tentative = false; - - var foundEntry = default(HashBucketEntry); - foundEntry.word = Interlocked.CompareExchange( - ref bucket->bucket_entries[slot], - updatedEntry.word, entry.word); - - if (foundEntry.word == entry.word) - { - goto LatchRelease; - } - else - { - // CAS failed - hlog.GetInfo(newPhysicalAddress).Invalid = true; - status = OperationStatus.RETRY_NOW; - goto LatchRelease; - } - } - #endregion - - #region Create failure context - CreateFailureContext: - { - pendingContext.type = OperationType.RMW; - pendingContext.key = hlog.GetKeyContainer(ref key); - pendingContext.input = input; - pendingContext.userContext = userContext; - pendingContext.entry.word = entry.word; - pendingContext.logicalAddress = logicalAddress; - pendingContext.version = sessionCtx.version; - pendingContext.serialNum = lsn; - pendingContext.heldLatch = heldOperation; - } - #endregion - - #region Latch release - LatchRelease: - { - switch (latchOperation) - { - case LatchOperation.Shared: - HashBucket.ReleaseSharedLatch(bucket); - break; - case LatchOperation.Exclusive: - HashBucket.ReleaseExclusiveLatch(bucket); - break; - default: - break; - } - } - #endregion - - return status == OperationStatus.RETRY_NOW - ? PsfInternalRMW(ref key, ref input, ref userContext, ref pendingContext, sessionCtx, lsn) - : status; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal OperationStatus PsfInternalDelete( // TODO - ref Key key, - ref Context userContext, - ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) - { - var status = default(OperationStatus); - var bucket = default(HashBucket*); - var slot = default(int); - var logicalAddress = Constants.kInvalidAddress; - var physicalAddress = default(long); - var latchOperation = default(LatchOperation); - var version = default(int); - var latestRecordVersion = -1; - - var hash = comparer.GetHashCode64(ref key); - var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); - - if (sessionCtx.phase != Phase.REST) - HeavyEnter(hash, sessionCtx); - - #region Trace back for record in in-memory HybridLog - var entry = default(HashBucketEntry); - var tagExists = FindTag(hash, tag, ref bucket, ref slot, ref entry); - if (!tagExists) - return OperationStatus.NOTFOUND; - - logicalAddress = entry.Address; - - if (UseReadCache) - SkipAndInvalidateReadCache(ref logicalAddress, ref latestRecordVersion, ref key); - var latestLogicalAddress = logicalAddress; - - if (logicalAddress >= hlog.ReadOnlyAddress) - { - physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - if (latestRecordVersion == -1) - latestRecordVersion = hlog.GetInfo(physicalAddress).Version; - if (!comparer.Equals(ref key, ref hlog.GetKey(physicalAddress))) - { - logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; - TraceBackForKeyMatch(ref key, - logicalAddress, - hlog.ReadOnlyAddress, - out logicalAddress, - out physicalAddress); - } - } - #endregion - - // NO optimization for most common case - //if (sessionCtx.phase == Phase.REST && logicalAddress >= hlog.ReadOnlyAddress) - //{ - // hlog.GetInfo(physicalAddress).Tombstone = true; - // return OperationStatus.SUCCESS; - //} - - #region Entry latch operation - if (sessionCtx.phase != Phase.REST) - { - switch (sessionCtx.phase) - { - case Phase.PREPARE: - { - version = sessionCtx.version; - if (HashBucket.TryAcquireSharedLatch(bucket)) - { - // Set to release shared latch (default) - latchOperation = LatchOperation.Shared; - if (latestRecordVersion != -1 && latestRecordVersion > version) - { - status = OperationStatus.CPR_SHIFT_DETECTED; - goto CreatePendingContext; // Pivot Thread - } - break; // Normal Processing - } - else - { - status = OperationStatus.CPR_SHIFT_DETECTED; - goto CreatePendingContext; // Pivot Thread - } - } - case Phase.IN_PROGRESS: - { - version = (sessionCtx.version - 1); - if (latestRecordVersion != -1 && latestRecordVersion <= version) - { - if (HashBucket.TryAcquireExclusiveLatch(bucket)) - { - // Set to release exclusive latch (default) - latchOperation = LatchOperation.Exclusive; - goto CreateNewRecord; // Create a (v+1) record - } - else - { - status = OperationStatus.RETRY_LATER; - goto CreatePendingContext; // Go Pending - } - } - break; // Normal Processing - } - case Phase.WAIT_PENDING: - { - version = (sessionCtx.version - 1); - if (latestRecordVersion != -1 && latestRecordVersion <= version) - { - if (HashBucket.NoSharedLatches(bucket)) - { - goto CreateNewRecord; // Create a (v+1) record - } - else - { - status = OperationStatus.RETRY_LATER; - goto CreatePendingContext; // Go Pending - } - } - break; // Normal Processing - } - case Phase.WAIT_FLUSH: - { - version = (sessionCtx.version - 1); - if (latestRecordVersion != -1 && latestRecordVersion <= version) - { - goto CreateNewRecord; // Create a (v+1) record - } - break; // Normal Processing - } - default: - break; - } - } - #endregion - - Debug.Assert(latestRecordVersion <= sessionCtx.version); - - #region Normal processing - - // Record is in memory, try to update hash chain and completely elide record - // only if previous address points to invalid address - if (logicalAddress >= hlog.ReadOnlyAddress) - { - if (entry.Address == logicalAddress && hlog.GetInfo(physicalAddress).PreviousAddress < hlog.BeginAddress) - { - var updatedEntry = default(HashBucketEntry); - updatedEntry.Tag = 0; - if (hlog.GetInfo(physicalAddress).PreviousAddress == Constants.kTempInvalidAddress) - updatedEntry.Address = Constants.kInvalidAddress; - else - updatedEntry.Address = hlog.GetInfo(physicalAddress).PreviousAddress; - updatedEntry.Pending = entry.Pending; - updatedEntry.Tentative = false; - - if (entry.word == Interlocked.CompareExchange(ref bucket->bucket_entries[slot], updatedEntry.word, entry.word)) - { - // Apply tombstone bit to the record - hlog.GetInfo(physicalAddress).Tombstone = true; - - if (WriteDefaultOnDelete) - { - // Write default value - // Ignore return value, the record is already marked - Value v = default; - functions.ConcurrentWriter(ref hlog.GetKey(physicalAddress), ref v, ref hlog.GetValue(physicalAddress)); - } - - status = OperationStatus.SUCCESS; - goto LatchRelease; // Release shared latch (if acquired) - } - } - } - - // Mutable Region: Update the record in-place - if (logicalAddress >= hlog.ReadOnlyAddress) - { - hlog.GetInfo(physicalAddress).Tombstone = true; - - if (WriteDefaultOnDelete) - { - // Write default value - // Ignore return value, the record is already marked - Value v = default; - functions.ConcurrentWriter(ref hlog.GetKey(physicalAddress), ref v, ref hlog.GetValue(physicalAddress)); - } - - status = OperationStatus.SUCCESS; - goto LatchRelease; // Release shared latch (if acquired) - } - - // All other regions: Create a record in the mutable region - #endregion - - #region Create new record in the mutable region - CreateNewRecord: - { - var value = default(Value); - // Immutable region or new record - // Allocate default record size for tombstone - var recordSize = hlog.GetRecordSize(ref key, ref value); - BlockAllocate(recordSize, out long newLogicalAddress, sessionCtx); - var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); - RecordInfo.WriteInfo(ref hlog.GetInfo(newPhysicalAddress), - sessionCtx.version, - true, true, false, - latestLogicalAddress); - hlog.ShallowCopy(ref key, ref hlog.GetKey(newPhysicalAddress)); - - var updatedEntry = default(HashBucketEntry); - updatedEntry.Tag = tag; - updatedEntry.Address = newLogicalAddress & Constants.kAddressMask; - updatedEntry.Pending = entry.Pending; - updatedEntry.Tentative = false; - - var foundEntry = default(HashBucketEntry); - foundEntry.word = Interlocked.CompareExchange( - ref bucket->bucket_entries[slot], - updatedEntry.word, entry.word); - - if (foundEntry.word == entry.word) - { - status = OperationStatus.SUCCESS; - goto LatchRelease; - } - else - { - hlog.GetInfo(newPhysicalAddress).Invalid = true; - status = OperationStatus.RETRY_NOW; - goto LatchRelease; - } - } - #endregion - - #region Create pending context - CreatePendingContext: - { - pendingContext.type = OperationType.DELETE; - pendingContext.key = hlog.GetKeyContainer(ref key); - pendingContext.userContext = userContext; - pendingContext.entry.word = entry.word; - pendingContext.logicalAddress = logicalAddress; - pendingContext.version = sessionCtx.version; - pendingContext.serialNum = lsn; - } - #endregion - - #region Latch release - LatchRelease: - { - switch (latchOperation) - { - case LatchOperation.Shared: - HashBucket.ReleaseSharedLatch(bucket); - break; - case LatchOperation.Exclusive: - HashBucket.ReleaseExclusiveLatch(bucket); - break; - default: - break; - } - } - #endregion - - return status == OperationStatus.RETRY_NOW - ? PsfInternalDelete(ref key, ref userContext, ref pendingContext, sessionCtx, lsn) - : status; - } } } \ No newline at end of file diff --git a/cs/src/core/Index/PSF/PSFQuerySession.cs b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs similarity index 91% rename from cs/src/core/Index/PSF/PSFQuerySession.cs rename to cs/src/core/Index/PSF/FasterPSFSessionOperations.cs index 3b505473f..172a26c8c 100644 --- a/cs/src/core/Index/PSF/PSFQuerySession.cs +++ b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs @@ -55,6 +55,39 @@ internal Status PsfReadAddress(ref PSFReadArgs psfArgs, long serialN if (SupportAsync) UnsafeSuspendThread(); } } + + internal Status PsfUpdate(ref GroupKeysPair groupKeysPair, ref Value value, ref Input input, long serialNo, + PSFChangeTracker changeTracker) + where TRecordId : struct + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return fht.ContextPsfUpdate(ref groupKeysPair, ref value, ref input, serialNo, ctx, changeTracker); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + internal Status PsfDelete(ref Key key, ref Value value, ref Input input, long serialNo, + PSFChangeTracker changeTracker) + where TRecordId : struct + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return fht.ContextPsfDelete(ref key, ref value, ref input, serialNo, ctx, changeTracker); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + #endregion PSF calls for Secondary FasterKV #region PSF Query API for primary FasterKV @@ -124,7 +157,7 @@ public IEnumerable> QueryPSF /// The type of the key value for the first /// The type of the key value for the second /// The first Predicate Subset Function object - /// The secojnd Predicate Subset Function object + /// The second Predicate Subset Function object /// The key value to return results from the first 's stored values /// The key value to return results from the second 's stored values /// A predicate that takes as parameters 1) whether a candidate record matches diff --git a/cs/src/core/Index/PSF/GroupKeys.cs b/cs/src/core/Index/PSF/GroupKeys.cs new file mode 100644 index 000000000..ab06d3974 --- /dev/null +++ b/cs/src/core/Index/PSF/GroupKeys.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + internal struct GroupKeys : IDisposable + { + // This cannot be typed to a TPSFKey because there may be different TPSFKeys across groups. + private SectorAlignedMemory compositeKeyMem; + private SectorAlignedMemory flagsMem; + + internal void Set(SectorAlignedMemory keyMem, SectorAlignedMemory flagsMem) + { + this.compositeKeyMem = keyMem; + this.flagsMem = flagsMem; + } + + internal unsafe ref TCompositeKey GetCompositeKeyRef() + => ref Unsafe.AsRef(this.compositeKeyMem.GetValidPointer()); + + internal unsafe PSFResultFlags* ResultFlags => (PSFResultFlags*)this.flagsMem.GetValidPointer(); + + internal unsafe bool IsNullAt(int ordinal) => (this.ResultFlags + ordinal)->HasFlag(PSFResultFlags.IsNull); + public unsafe bool IsUnlinkOldAt(int ordinal) => (this.ResultFlags + ordinal)->HasFlag(PSFResultFlags.UnlinkOld); + public unsafe bool IsLinkNewAt(int ordinal) => (this.ResultFlags + ordinal)->HasFlag(PSFResultFlags.LinkNew); + + public unsafe bool HasChanges => this.ResultFlags->HasFlag(PSFResultFlags.UnlinkOld) + || this.ResultFlags->HasFlag(PSFResultFlags.LinkNew); + + public void Dispose() + { + this.compositeKeyMem.Return(); + this.flagsMem.Return(); + } + } + + internal struct GroupKeysPair : IDisposable + { + internal long GroupId; + internal int KeySize; + + // If the PSFGroup found the RecordId in its IPUCache, we carry it here. + internal long LogicalAddress; + + internal GroupKeys Before; + internal GroupKeys After; + + internal bool HasAddress => this.LogicalAddress != Constants.kInvalidAddress; + + internal ref TCompositeKey GetBeforeKey() + => ref this.Before.GetCompositeKeyRef(); + + internal ref TCompositeKey GetAfterKey() + => ref this.After.GetCompositeKeyRef(); + + internal bool HasChanges => this.After.HasChanges; + + public void Dispose() + { + this.Before.Dispose(); + this.After.Dispose(); + } + } +} diff --git a/cs/src/core/Index/PSF/IChainPost.cs b/cs/src/core/Index/PSF/IChainPost.cs index 365d6b216..e6fff063e 100644 --- a/cs/src/core/Index/PSF/IChainPost.cs +++ b/cs/src/core/Index/PSF/IChainPost.cs @@ -3,6 +3,11 @@ namespace FASTER.core { + /// + /// This represents (a generic decoupling of) the concept that each PSFValue is essentially a "post" + /// that supports the chains, like a post in a barbed-wire fence. + /// + /// internal unsafe interface IChainPost { int ChainHeight { get; } @@ -10,5 +15,7 @@ internal unsafe interface IChainPost int RecordIdSize { get; } long* GetChainLinkPtrs(ref TPSFValue value); + + void SetRecordId(ref TPSFValue value, TRecordId recordId); } } \ No newline at end of file diff --git a/cs/src/core/Index/PSF/IExecutePSF.cs b/cs/src/core/Index/PSF/IExecutePSF.cs index c580d0ba6..bdc689dfb 100644 --- a/cs/src/core/Index/PSF/IExecutePSF.cs +++ b/cs/src/core/Index/PSF/IExecutePSF.cs @@ -13,6 +13,7 @@ namespace FASTER.core /// /// public interface IExecutePSF + where TRecordId : struct { /// /// For each in the , @@ -20,8 +21,11 @@ public interface IExecutePSF /// /// The provider's data, e.g. /// The provider's record ID, e.g. long (logicalAddress) for FasterKV - /// - Status ExecuteAndStore(TProviderData data, TRecordId recordId, bool isInserted); + /// The phase of PSF operations in which this execution is being done + /// Tracks the values for comparison + /// to the values + Status ExecuteAndStore(TProviderData data, TRecordId recordId, PSFExecutePhase phase, + PSFChangeTracker changeTracker); /// /// For the given , verify that the @@ -33,5 +37,28 @@ public interface IExecutePSF /// True if the providerData matches (returns non-null from) the , /// else false. bool Verify(TProviderData providerData, int psfOrdinal); + + /// + /// The identifier of this . + /// + long Id { get; } + + /// + /// Get the TPSFKeys for the current (before updating) state of the RecordId + /// The record of previous key values and updated values + /// + Status GetBeforeKeys(PSFChangeTracker changeTracker); + + /// + /// Update the RecordId + /// The record of previous key values and updated values + /// + Status Update(PSFChangeTracker changeTracker); + + /// + /// Delete the RecordId + /// The record of previous key values and updated values + /// + Status Delete(PSFChangeTracker changeTracker); } } diff --git a/cs/src/core/Index/PSF/PSF.cs b/cs/src/core/Index/PSF/PSF.cs index 8f5fcce8e..9f3e79025 100644 --- a/cs/src/core/Index/PSF/PSF.cs +++ b/cs/src/core/Index/PSF/PSF.cs @@ -16,16 +16,16 @@ public class PSF : IPSF { private readonly IQueryPSF psfGroup; - internal int GroupOrdinal { get; } // in the psfGroup list + internal long GroupId { get; } // unique in the PSFManager.psfGroup list internal int PsfOrdinal { get; } // in the psfGroup /// public string Name { get; } - internal PSF(int groupOrdinal, int psfOrdinal, string name, IQueryPSF iqp) + internal PSF(long groupId, int psfOrdinal, string name, IQueryPSF iqp) { - this.GroupOrdinal = groupOrdinal; + this.GroupId = groupId; this.PsfOrdinal = psfOrdinal; this.Name = name; this.psfGroup = iqp; diff --git a/cs/src/core/Index/PSF/PSFChangeTracker.cs b/cs/src/core/Index/PSF/PSFChangeTracker.cs new file mode 100644 index 000000000..982ab53bb --- /dev/null +++ b/cs/src/core/Index/PSF/PSFChangeTracker.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + public enum UpdateOperation + { + Insert, + IPU, + RCU, + Delete + } + + public unsafe class PSFChangeTracker : IDisposable + where TRecordId : struct + { + #region External API + public TProviderData BeforeData { get; set; } + public TRecordId BeforeRecordId { get; set; } + + public TProviderData AfterData { get; set; } + public TRecordId AfterRecordId { get; set; } + + public UpdateOperation UpdateOp { get; set; } + #endregion External API + + private GroupKeysPair[] groups; + + internal long CachedBeforeLA = Constants.kInvalidAddress; + + internal void PrepareGroups(int numGroups) + { + this.groups = new GroupKeysPair[numGroups]; + for (var ii = 0; ii < numGroups; ++ii) + this.groups[ii].GroupId = Constants.kInvalidPsfGroupId; + } + + internal bool FindGroup(long groupId, out int ordinal) + { + for (var ii = 0; ii < this.groups.Length; ++ii) // TODO will there be enough groups for sequential search to matter? + { + if (groups[ii].GroupId == groupId) + { + ordinal = ii; + return true; + } + } + + // Likely the groupId was from a group added since this PSFChangeTracker instance was created. + ordinal = -1; + return false; + } + + internal ref GroupKeysPair GetGroupRef(int ordinal) + => ref groups[ordinal]; + + internal ref GroupKeysPair FindFreeGroupRef(long groupId, int keySize, long logAddr = Constants.kInvalidAddress) + { + if (!this.FindGroup(Constants.kInvalidPsfGroupId, out var ordinal)) + { + // A new group was added while we were populating this; should be quite rare. // TODO test this case + var groups = new GroupKeysPair[this.groups.Length + 1]; + Array.Copy(this.groups, groups, this.groups.Length); + this.groups = groups; + ordinal = this.groups.Length - 1; + } + ref GroupKeysPair ret = ref this.groups[ordinal]; + ret.GroupId = groupId; + ret.KeySize = keySize; + ret.LogicalAddress = logAddr; + return ref ret; + } + + public void AssignRcuRecordId(ref PSFValue value) + => value.RecordId = this.AfterRecordId; + + public void Dispose() + { + foreach (var group in this.groups) + group.Dispose(); + } + } +} diff --git a/cs/src/core/Index/PSF/PSFCompositeKey.cs b/cs/src/core/Index/PSF/PSFCompositeKey.cs index dc41e9078..d7394e7d0 100644 --- a/cs/src/core/Index/PSF/PSFCompositeKey.cs +++ b/cs/src/core/Index/PSF/PSFCompositeKey.cs @@ -73,7 +73,7 @@ internal unsafe struct PtrWrapper : IDisposable internal PtrWrapper(int size, SectorAlignedBufferPool pool) { this.size = size; - mem = pool.Get(size); + this.mem = pool.Get(size); } internal void Set(ref TPSFKey key) diff --git a/cs/src/core/Index/PSF/PSFExecutePhase.cs b/cs/src/core/Index/PSF/PSFExecutePhase.cs new file mode 100644 index 000000000..303094e53 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFExecutePhase.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// The phase of PSF operations in which PSFs are being executed + /// + public enum PSFExecutePhase + { + /// + /// Executing the PSFs to obtain new TPSFKeys to be inserted into the secondary FKV. + /// + Insert, + + /// + /// Lookup in IPUCache or execute the PSFs to obtain TPSFKeys prior to update, to be compared to those + /// obtained after the update to modify the record's PSF membership in the secondary FKV. + /// + PreUpdate, + + /// + /// Executing the PSFs to obtain TPSFKeys following an update, to be compared to those obtained before + /// the update to modify the record's PSF membership in the secondary FKV. + /// + PostUpdate, + + /// + /// Lookup in IPUCache to tombstone a record, or execute the PSFs to obtain TPSFKeys to place a new + /// tombstoned record. + /// + Delete + } +} diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index a33a4a9ae..4982c9a86 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Runtime.CompilerServices; @@ -22,13 +23,18 @@ public class PSFGroup : IExecutePSF, PSFValue, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions> fht; private readonly PSFFunctions functions; internal IPSFDefinition[] psfDefinitions; - private readonly int ordinal; // in the parent's PSFGroup list TODO needed? private readonly PSFRegistrationSettings regSettings; + /// + /// ID of the group (used internally only) + /// + public long Id { get; } + private readonly CheckpointSettings checkpointSettings; private readonly int keySize = Utility.GetSize(default(TPSFKey)); private readonly int recordIdSize = (Utility.GetSize(default(TRecordId)) + sizeof(long) - 1) & ~(sizeof(long) - 1); @@ -59,31 +65,42 @@ PSFOutputSecondary, PSFContext, PSFFunctions + public override int GetHashCode() => this.Id.GetHashCode(); + + /// + public override bool Equals(object obj) => this.Equals(obj as PSFGroup); + + /// + public bool Equals(PSFGroup other) => other is null ? false : this.Id == other.Id; + /// /// Constructor /// - /// The ordinal of this PSFGroup in the 's + /// The ordinal of this PSFGroup in the 's /// PSFGroup list. /// PSF definitions /// Optional registration settings - public PSFGroup(int ordinal, IPSFDefinition[] defs, PSFRegistrationSettings regSettings) + public PSFGroup(long id, IPSFDefinition[] defs, PSFRegistrationSettings regSettings) { // TODO check for null defs regSettings etc; for PSFs defined on a FasterKV instance we create intelligent // defaults in regSettings but other clients will have to specify at least hashTableSize, logSettings, etc. this.psfDefinitions = defs; - this.ordinal = ordinal; + this.Id = id; this.regSettings = regSettings; this.userKeyComparer = GetUserKeyComparer(); - this.PSFs = defs.Select((def, ord) => new PSF(this.ordinal, ord, def.Name, this)).ToArray(); + this.PSFs = defs.Select((def, ord) => new PSF(this.Id, ord, def.Name, this)).ToArray(); this.compositeKeyComparer = new PSFCompositeKeyComparer(this.userKeyComparer, this.keySize, this.ChainHeight); // TODO doc (or change) chainPost terminology - this.chainPost = new PSFValue.ChainPost(this.ChainHeight, this.keySize); + this.chainPost = new PSFValue.ChainPost(this.ChainHeight, this.recordIdSize); this.checkpointSettings = regSettings?.CheckpointSettings; this.functions = new PSFFunctions(chainPost); - this.fht = new FasterKV, PSFValue, PSFInputSecondary, + this.fht = new FasterKV, PSFValue, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions>( regSettings.HashTableSize, new PSFFunctions(chainPost), regSettings.LogSettings, this.checkpointSettings, null /*SerializerSettings*/, @@ -102,12 +119,12 @@ public PSFGroup(int ordinal, IPSFDefinition[] defs, PSFR [MethodImpl(MethodImplOptions.AggressiveInlining)] private ClientSession, PSFValue, PSFInputSecondary, PSFOutputSecondary, PSFContext, - PSFFunctions> GetSession () + PSFFunctions> GetSession() { // Sessions are used only on post-RegisterPSF actions (Upsert, RMW, Query). if (this.freeSessions.TryPop(out var session)) return session; - session = this.fht.NewSession(threadAffinitized:this.regSettings.ThreadAffinitized); + session = this.fht.NewSession(threadAffinitized: this.regSettings.ThreadAffinitized); this.allSessions.Add(session); return session; } @@ -141,28 +158,54 @@ public PSF this[string name] => Array.Find(this.PSFs, psf => psf.Name.Equals(name, StringComparison.CurrentCultureIgnoreCase)) ?? throw new InvalidOperationException("TODO PSF Exception classes: PSF not found"); + private unsafe void StoreKeys(ref GroupKeys keys, byte* kPtr, int kLen, PSFResultFlags* flagsPtr, int flagsLen) + { + var poolKeyMem = this.bufferPool.Get(kLen); + Buffer.MemoryCopy(kPtr, poolKeyMem.GetValidPointer(), kLen, kLen); + var flagsMem = this.bufferPool.Get(flagsLen); + Buffer.MemoryCopy(flagsPtr, flagsMem.GetValidPointer(), flagsLen, flagsLen); + keys.Set(poolKeyMem, flagsMem); + } + + internal unsafe void MarkChanges(GroupKeysPair keysPair) + { + var before = keysPair.Before; + var after = keysPair.After; + var beforeCompKey = before.GetCompositeKeyRef>(); + var afterCompKey = after.GetCompositeKeyRef>(); + for (var ii = 0; ii < this.ChainHeight; ++ii) + { + bool keysEqual() => beforeCompKey.GetKeyRef(ii, this.keySize).Equals(afterCompKey.GetKeyRef(ii, this.keySize)); + + // IsNull is already set in PSFGroup.ExecuteAndStore. + if (!before.IsNullAt(ii) && (after.IsNullAt(ii) || !keysEqual())) + *after.ResultFlags |= PSFResultFlags.UnlinkOld; + if (!after.IsNullAt(ii) && (before.IsNullAt(ii) || !keysEqual())) + *after.ResultFlags |= PSFResultFlags.LinkNew; + } + } + /// - public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recordId, bool isInserted) + public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recordId, PSFExecutePhase phase, + PSFChangeTracker changeTracker) { - // Note: stackalloc is safe here because PendingContext will copy it to the bufferPool if needed. - var keyMemLen = ((keySize * this.ChainHeight + sizeof(int) - 1) & ~(sizeof(int) - 1)) / sizeof(int); - var keyMem = stackalloc int[keyMemLen]; + // Note: stackalloc is safe because PendingContext or PSFChangeTracker will copy it to the bufferPool + // if needed. On the Insert fast path, we don't want any allocations otherwise; changeTracker is null. + // TODO: Change ChainHeight to psfCount + var keyMemLen = ((keySize * this.ChainHeight + sizeof(int) - 1) & ~(sizeof(int) - 1)); + var keyMemInt = stackalloc int[keyMemLen / sizeof(int)]; for (var ii = 0; ii < keyMemLen; ++ii) - keyMem[ii] = 0; + keyMemInt[ii] = 0; + var keyMem = (byte*)keyMemInt; ref PSFCompositeKey compositeKey = ref Unsafe.AsRef>(keyMem); - var valueMem = stackalloc byte[recordIdSize + sizeof(long) * this.ChainHeight]; - ref PSFValue psfValue = ref Unsafe.AsRef>(valueMem); - psfValue.RecordId = recordId; - - bool* nullIndicators = stackalloc bool[this.ChainHeight]; + var flagsMemLen = this.ChainHeight * sizeof(PSFResultFlags); + PSFResultFlags* flags = stackalloc PSFResultFlags[this.ChainHeight]; var anyMatch = false; - long* chainLinkPtrs = this.fht.chainPost.GetChainLinkPtrs(ref psfValue); for (int ii = 0; ii < this.psfDefinitions.Length; ++ii) { var key = this.psfDefinitions[ii].Execute(providerData); - *(nullIndicators + ii) = !key.HasValue; - *(chainLinkPtrs + ii) = Constants.kInvalidAddress; + *(flags + ii) = key.HasValue ? PSFResultFlags.None : PSFResultFlags.IsNull; if (key.HasValue) { ref TPSFKey keyPtr = ref Unsafe.AsRef(keyMem + keySize * ii); @@ -171,28 +214,122 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor } } - if (!anyMatch) + if (!anyMatch && phase == PSFExecutePhase.Insert) return Status.OK; - var input = new PSFInputSecondary(0, compositeKeyComparer, nullIndicators); - var session = this.GetSession(); - Status status; - try // TODOperf: Assess overhead of try/finally + var input = new PSFInputSecondary(0, compositeKeyComparer, this.Id, flags); + + var valueMem = stackalloc byte[recordIdSize + sizeof(long) * this.ChainHeight]; + ref PSFValue psfValue = ref Unsafe.AsRef>(valueMem); + psfValue.RecordId = recordId; + + int groupOrdinal = -1; + if (!(changeTracker is null)) { - status = isInserted - ? session.PsfInsert(ref compositeKey, ref psfValue, ref input, 1 /*TODO: lsn*/) - : throw new NotImplementedException("TODO updates"); + psfValue.RecordId = changeTracker.BeforeRecordId; + if (phase == PSFExecutePhase.PreUpdate) + { + // Get a free group ref and store the "before" values. + ref GroupKeysPair groupKeysPair = ref changeTracker.FindFreeGroupRef(this.Id, this.keySize); + StoreKeys(ref groupKeysPair.Before, keyMem, keyMemLen, flags, flagsMemLen); + return Status.OK; + } + + if (phase == PSFExecutePhase.PostUpdate) + { + // If not found, this is a new group added after the PreUpdate was done, so handle this as an insert. + if (!changeTracker.FindGroup(this.Id, out groupOrdinal)) + { + phase = PSFExecutePhase.Insert; + } + else + { + ref GroupKeysPair groupKeysPair = ref changeTracker.GetGroupRef(groupOrdinal); + StoreKeys(ref groupKeysPair.After, keyMem, keyMemLen, flags, flagsMemLen); + this.MarkChanges(groupKeysPair); + // TODO in debug, for initial dev, follow chains to assert the values match what is in the record's compositeKey + if (!groupKeysPair.HasChanges) + return Status.OK; + } + } + + // We don't need to do anything here for Delete. } + + long* chainLinkPtrs = this.fht.chainPost.GetChainLinkPtrs(ref psfValue); + for (int ii = 0; ii < this.psfDefinitions.Length; ++ii) + *(chainLinkPtrs + ii) = Constants.kInvalidAddress; + + var session = this.GetSession(); + try + { + return phase switch + { + PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey, ref psfValue, ref input, 1 /*TODO: lsn*/), + PSFExecutePhase.PostUpdate => session.PsfUpdate(ref changeTracker.GetGroupRef(groupOrdinal), ref psfValue, ref input, 1 /*TODO: lsn*/, + changeTracker), + PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey, ref psfValue, ref input, 1 /*TODO: lsn*/, + changeTracker), + _ => throw new FasterException("TODO: Make this a FasterInvalidOperationException") + }; + } finally { this.ReleaseSession(session); } - return status; + } + + /// + public Status GetBeforeKeys(PSFChangeTracker changeTracker) + { + // Obtain the "before" values. TODO try to find TRecordId in the IPUCache first. + return ExecuteAndStore(changeTracker.BeforeData, default, PSFExecutePhase.PreUpdate, changeTracker); + } + + /// + /// Update the RecordId + /// + public Status Update(PSFChangeTracker changeTracker) + { + changeTracker.CachedBeforeLA = Constants.kInvalidAddress; // TODO Find BeforeRecordId in IPUCache + if (changeTracker.CachedBeforeLA != Constants.kInvalidAddress) + { + if (changeTracker.UpdateOp == UpdateOperation.RCU) + { + // TODO: Tombstone it, and possibly unlink; or just copy its keys into changeTracker.Before. + } + else + { + // TODO: Try to splice in-place; or just copy its keys into changeTracker.Before. + } + } + else + { + if (this.GetBeforeKeys(changeTracker) != Status.OK) + { + // TODO handle errors from GetBeforeKeys + } + } + return this.ExecuteAndStore(changeTracker.AfterData, default, PSFExecutePhase.PostUpdate, changeTracker); + } + + /// + /// Delete the RecordId + /// + public Status Delete(PSFChangeTracker changeTracker) + { + changeTracker.CachedBeforeLA = Constants.kInvalidAddress; // TODO Find BeforeRecordId in IPUCache + if (changeTracker.CachedBeforeLA != Constants.kInvalidAddress) + { + // TODO: Tombstone it, and possibly unlink; or just copy its keys into changeTracker.Before. + // If the latter, we can bypass ExecuteAndStore's PSF-execute loop + } + return this.ExecuteAndStore(changeTracker.BeforeData, default, PSFExecutePhase.Delete, changeTracker); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool Verify(TProviderData providerData, int psfOrdinal) + public bool Verify(TProviderData providerData, int psfOrdinal) // TODO revise to new spec => !(this.psfDefinitions[psfOrdinal].Execute(providerData) is null); /// @@ -209,7 +346,7 @@ public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key) // having to add additional generic args. Interfaces on structs incur boxing overhead (plus the complexity // of mutable structs). But check the performance here; if necessary perhaps I can figure out a way to // pass a struct with no TRecordId, TPSFKey, etc. and use an FHT-level interface to manage it. - var psfInput = new PSFInputSecondary(psfOrdinal, compositeKeyComparer); + var psfInput = new PSFInputSecondary(psfOrdinal, compositeKeyComparer, this.Id); return Query(keyPtr, psfInput); } @@ -221,6 +358,7 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe var session = this.GetSession(); Status status; + HashSet deadRecs = null; try { status = session.PsfReadKey(ref queryKeyPtr.GetRef(), ref readArgs, 1 /*TODO lsn*/); @@ -228,7 +366,16 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe yield break; var secondaryOutput = readArgs.Output as IPSFSecondaryOutput; - yield return secondaryOutput.RecordId; + + if (secondaryOutput.Tombstone) + { + deadRecs ??= new HashSet(); + deadRecs.Add(secondaryOutput.RecordId); + } + else + { + yield return secondaryOutput.RecordId; + } do { @@ -236,6 +383,20 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe status = session.PsfReadAddress(ref readArgs, 1 /*TODO lsn*/); if (status != Status.OK) // TODO check other status yield break; + + if (secondaryOutput.Tombstone) + { + deadRecs ??= new HashSet(); + deadRecs.Add(secondaryOutput.RecordId); + continue; + } + + if (!(deadRecs is null) && deadRecs.Contains(secondaryOutput.RecordId)) + { + if (!secondaryOutput.Tombstone) // A live record will not be encountered again so remove it + deadRecs.Remove(secondaryOutput.RecordId); + continue; + } yield return secondaryOutput.RecordId; } while (secondaryOutput.PreviousLogicalAddress != Constants.kInvalidAddress); } @@ -244,7 +405,6 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe this.ReleaseSession(session); queryKeyPtr.Dispose(); } - } } } diff --git a/cs/src/core/Index/PSF/PSFInputSecondary.cs b/cs/src/core/Index/PSF/PSFInput.cs similarity index 70% rename from cs/src/core/Index/PSF/PSFInputSecondary.cs rename to cs/src/core/Index/PSF/PSFInput.cs index ad8700a4b..d32de2197 100644 --- a/cs/src/core/Index/PSF/PSFInputSecondary.cs +++ b/cs/src/core/Index/PSF/PSFInput.cs @@ -13,8 +13,16 @@ namespace FASTER.core /// /// The type of the Key, either a for the /// secondary FasterKV instances, or the user's TKVKey for the primary FasterKV instance. + /// The interface separation is needed for the PendingContext public interface IPSFInput { + unsafe void SetFlags(PSFResultFlags* resultFlags); + + /// + /// The ID of the for this operation. + /// + long GroupId { get; } + /// /// The ordinal of the being queried; writable only for Insert. /// @@ -26,6 +34,11 @@ public interface IPSFInput /// bool IsNullAt { get; } + /// + /// For Delete() or Insert() done as part of RCU, this indicates if it the tombstone should be set for this record. + /// + bool IsDelete { get; set; } + /// /// For tracing back the chain, this is the next logicalAddress to get. /// @@ -60,22 +73,37 @@ internal PSFInputPrimaryReadAddress(long readLA) this.ReadLogicalAddress = readLA; } + // TODO: Trim this class and IPSFInput down to just ReadLogicalAddress.. casting to SecondaryInput + // in PsfInternalInsert should be sufficient + + /// + public long GroupId => throw new InvalidOperationException("Not valid for Primary FKV"); + /// public int PsfOrdinal { get => Constants.kInvalidPsfOrdinal; - set => throw new InvalidOperationException("Not valid for reading from Primary Address"); + set => throw new InvalidOperationException("Not valid for Primary FKV"); } - public bool IsNullAt => throw new InvalidOperationException("Not valid for reading from Primary Address"); + public void SetFlags(PSFResultFlags* resultFlags) + => throw new InvalidOperationException("Not valid for Primary FKV"); + + public bool IsNullAt => throw new InvalidOperationException("Not valid for Primary FKV"); + + public bool IsDelete + { + get => throw new InvalidOperationException("Not valid for Primary FKV"); + set => throw new InvalidOperationException("Not valid for Primary FKV"); + } public long ReadLogicalAddress { get; set; } public long GetHashCode64At(ref TKey key) - => throw new InvalidOperationException("Not valid for reading from Primary Address"); + => throw new InvalidOperationException("Not valid for Primary FKV"); public bool EqualsAt(ref TKey queryKey, ref TKey storedKey) - => throw new InvalidOperationException("Not valid for reading from Primary Address"); + => throw new InvalidOperationException("Not valid for Primary FKV"); } /// @@ -86,20 +114,27 @@ public unsafe class PSFInputSecondary : IPSFInput> comparer; - internal readonly bool* nullIndicators; + internal PSFResultFlags* resultFlags; internal PSFInputSecondary(int psfOrd, ICompositeKeyComparer> keyCmp, - bool* nullIndicators = null) + long groupId, PSFResultFlags* flags = null) { this.PsfOrdinal = psfOrd; this.comparer = keyCmp; - this.nullIndicators = nullIndicators; + this.GroupId = groupId; + this.resultFlags = flags; this.ReadLogicalAddress = Constants.kInvalidAddress; } + public long GroupId { get; } + public int PsfOrdinal { get; set; } - public bool IsNullAt => this.nullIndicators[this.PsfOrdinal]; + public void SetFlags(PSFResultFlags* resultFlags) => this.resultFlags = resultFlags; + + public bool IsNullAt => this.resultFlags[this.PsfOrdinal].HasFlag(PSFResultFlags.IsNull); + + public bool IsDelete { get; set; } public long ReadLogicalAddress { get; set; } @@ -109,6 +144,6 @@ public long GetHashCode64At(ref PSFCompositeKey cKey) [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool EqualsAt(ref PSFCompositeKey queryKey, ref PSFCompositeKey storedKey) - => this.comparer.Equals(this.nullIndicators is null, this.PsfOrdinal, ref queryKey, ref storedKey); + => this.comparer.Equals(this.resultFlags is null, this.PsfOrdinal, ref queryKey, ref storedKey); } } diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index 5c885b857..d338862ce 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -3,22 +3,64 @@ using System; using System.Collections.Generic; +using System.Threading; namespace FASTER.core { internal class PSFManager where TRecordId : struct, IComparable { - readonly List> psfGroups = new List>(); + readonly Dictionary> psfGroups + = new Dictionary>(); internal bool HasPSFs => this.psfGroups.Count > 0; - internal Status Upsert(TProviderData providerData, TRecordId recordId, bool isInserted) + internal Status Upsert(TProviderData data, TRecordId recordId, + PSFChangeTracker changeTracker) { // TODO: This is called from ContextUpsert or InternalCompleteRetryRequest, where it's still // in the Context threading control for the primaryKV. I think it needs to move out of there. - foreach (var group in this.psfGroups) + + // This Upsert was an Insert: For the FasterKV Insert fast path, changeTracker is null. + if (changeTracker is null || changeTracker.UpdateOp == UpdateOperation.Insert) + { + foreach (var group in this.psfGroups.Values) + { + // Fast Insert path: No IPUCache lookup is done for Inserts, so this is called directly here. + var status = group.ExecuteAndStore(data, recordId, PSFExecutePhase.Insert, changeTracker); + if (status != Status.OK) + { + // TODO handle errors + } + } + return Status.OK; + } + + // This Upsert was an IPU or RCU + return this.Update(changeTracker); + } + + internal Status Update(PSFChangeTracker changeTracker) + { + // TODO: same comment as Insert re: Context threading control for the primaryKV + changeTracker.PrepareGroups(this.psfGroups.Count); + foreach (var group in this.psfGroups.Values) { - var status = group.ExecuteAndStore(providerData, recordId, isInserted); + var status = group.Update(changeTracker); + if (status != Status.OK) + { + // TODO handle errors + } + } + return Status.OK; + } + + internal Status Delete(PSFChangeTracker changeTracker) + { + // TODO: same comment as Insert re: Context threading control for the primaryKV + changeTracker.PrepareGroups(this.psfGroups.Count); + foreach (var group in this.psfGroups.Values) + { + var status = group.Delete(changeTracker); if (status != Status.OK) { // TODO handle errors @@ -29,15 +71,26 @@ internal Status Upsert(TProviderData providerData, TRecordId recordId, bool isIn internal string[][] GetRegisteredPSFs() => throw new NotImplementedException("TODO"); + internal PSFChangeTracker CreateChangeTracker() + => new PSFChangeTracker(); + + private static long NextGroupId = 0; + + private void AddGroup(PSFGroup group) where TPSFKey : struct + { + var gId = Interlocked.Increment(ref NextGroupId); + this.psfGroups.Add(gId - 1, group); + } + internal PSF RegisterPSF(IPSFDefinition def, PSFRegistrationSettings registrationSettings) where TPSFKey : struct { // TODO: Runtime check that TPSFKey is blittable var group = new PSFGroup(this.psfGroups.Count, new[] { def }, registrationSettings); - this.psfGroups.Add(group); + AddGroup(group); return group[def.Name]; - } + } internal PSF[] RegisterPSF(IPSFDefinition[] defs, PSFRegistrationSettings registrationSettings) @@ -45,7 +98,7 @@ internal PSF[] RegisterPSF(IPSFDefinition(this.psfGroups.Count, defs, registrationSettings); - this.psfGroups.Add(group); + AddGroup(group); return group.PSFs; } @@ -55,8 +108,7 @@ internal IEnumerable QueryPSF(IPSFCreateProviderData QueryPSF(IPSFCreateProviderData QueryPSF( var e1done = !e1.MoveNext(); var e2done = !e2.MoveNext(); - var group1 = this.psfGroups[psf1.GroupOrdinal]; - var group2 = this.psfGroups[psf2.GroupOrdinal]; + var group1 = this.psfGroups[psf1.GroupId]; + var group2 = this.psfGroups[psf2.GroupId]; var cancelToken = querySettings is null ? default : querySettings.CancellationToken; bool cancellationRequested() diff --git a/cs/src/core/Index/PSF/PSFOutput.cs b/cs/src/core/Index/PSF/PSFOutput.cs index 6c9e1eef0..1d99a36fc 100644 --- a/cs/src/core/Index/PSF/PSFOutput.cs +++ b/cs/src/core/Index/PSF/PSFOutput.cs @@ -16,7 +16,7 @@ namespace FASTER.core /// secondary FasterKV instances, or the user's TKVValue for the primary FasterKV instance. public interface IPSFOutput { - PSFOperationStatus Visit(int psfOrdinal, ref TKey key, ref TValue value, bool isConcurrent); + PSFOperationStatus Visit(int psfOrdinal, ref TKey key, ref TValue value, bool tombstone, bool isConcurrent); } public interface IPSFPrimaryOutput @@ -46,9 +46,10 @@ internal PSFOutputPrimaryReadAddress(AllocatorBase alloc) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public PSFOperationStatus Visit(int psfOrdinal, ref TKVKey key, ref TKVValue value, bool isConcurrent) + public PSFOperationStatus Visit(int psfOrdinal, ref TKVKey key, ref TKVValue value, bool tombstone, bool isConcurrent) { - this.ProviderData = new FasterKVProviderData(allocator, ref key, ref value); + // tombstone is not used here; it is only needed for the chains in the secondary FKV. + this.ProviderData = new FasterKVProviderData(this.allocator, ref key, ref value); return new PSFOperationStatus(OperationStatus.SUCCESS); } } @@ -56,6 +57,9 @@ public PSFOperationStatus Visit(int psfOrdinal, ref TKVKey key, ref TKVValue val public interface IPSFSecondaryOutput { TRecordId RecordId { get; } + + bool Tombstone { get; } + long PreviousLogicalAddress { get; } } @@ -72,6 +76,9 @@ public unsafe class PSFOutputSecondary : IPSFOutput> chainPost; public TRecordId RecordId { get; private set; } + + public bool Tombstone { get; private set; } + public long PreviousLogicalAddress { get; private set; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -84,9 +91,10 @@ internal PSFOutputSecondary(IChainPost> chainPost) [MethodImpl(MethodImplOptions.AggressiveInlining)] public PSFOperationStatus Visit(int psfOrdinal, ref PSFCompositeKey key, - ref PSFValue value, bool isConcurrent) + ref PSFValue value, bool tombstone, bool isConcurrent) { this.RecordId = value.RecordId; + this.Tombstone = tombstone; this.PreviousLogicalAddress = *(this.chainPost.GetChainLinkPtrs(ref value) + psfOrdinal); return new PSFOperationStatus(OperationStatus.SUCCESS); } diff --git a/cs/src/core/Index/PSF/PSFRegistrationSettings.cs b/cs/src/core/Index/PSF/PSFRegistrationSettings.cs index d2be31c54..c10e96bf1 100644 --- a/cs/src/core/Index/PSF/PSFRegistrationSettings.cs +++ b/cs/src/core/Index/PSF/PSFRegistrationSettings.cs @@ -45,5 +45,16 @@ public class PSFRegistrationSettings /// Indicates whether PSFGroup Sessions are thread-affinitized. /// public bool ThreadAffinitized; + + /// + /// The size of the first IPU Cache; inserts are done into this cache only. If zero, no caching is done. + /// + public long IPU1CacheSize = 0; + + /// + /// The size of the second IPU Cache; inserts are not done into this cache, so more distant records + /// are likelier to remain. If this is nonzero, must also be nonzero. + /// + public long IPU2CacheSize = 0; } } diff --git a/cs/src/core/Index/PSF/PSFResultFlags.cs b/cs/src/core/Index/PSF/PSFResultFlags.cs new file mode 100644 index 000000000..b59998d60 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFResultFlags.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; + +namespace FASTER.core +{ + /// + /// Internal flags indicating how the keys returned from a are handled. + /// + [Flags] + public enum PSFResultFlags + { + /// + /// For initialization only + /// + None = 0, + + /// + /// For Insert, this identifies a null PSF result (the record does not match the PSF and is not + /// included in any TPSFKey chain for it). Also used in + /// to determine whether to set . + /// + IsNull = 1, + + /// + /// For Update, the TPSFKey has changed; remove this record from the previous TPSFKey chain. + /// + UnlinkOld = 2, + + /// + /// For Update and insert, the TPSFKey has changed; add this record to the new TPSFKey chain. + /// + LinkNew = 4 + } +} diff --git a/cs/src/core/Index/PSF/PSFUpdateArgs.cs b/cs/src/core/Index/PSF/PSFUpdateArgs.cs index b1cc3de2c..48821373c 100644 --- a/cs/src/core/Index/PSF/PSFUpdateArgs.cs +++ b/cs/src/core/Index/PSF/PSFUpdateArgs.cs @@ -3,9 +3,23 @@ namespace FASTER.core { - internal struct PSFUpdateArgs + /// + /// Structure passed to and set by Upsert and RMW in the Primary FKV; specific to the FasterKV client. + /// + internal struct PSFUpdateArgs + where Key : new() + where Value : new() { - internal long logicalAddress; // Set to the inserted or updated address - internal bool isInserted; // Set true if this was an insert rather than update or RMW + /// + /// Set to the inserted or updated logical address, which is the RecordId for the FasterKV client + /// + /// Having this outside the changeTracker means that a non-updating Upsert does not incur + /// allocation overhead. + internal long LogicalAddress; + + /// + /// Created and populated with PreUpdate values on RMW or Upsert of an existing key + /// + internal PSFChangeTracker, long> ChangeTracker; } } diff --git a/cs/src/core/Index/PSF/PSFValue.cs b/cs/src/core/Index/PSF/PSFValue.cs index efe7d9734..ef28dacb2 100644 --- a/cs/src/core/Index/PSF/PSFValue.cs +++ b/cs/src/core/Index/PSF/PSFValue.cs @@ -28,6 +28,10 @@ internal void CopyTo(ref PSFValue other, int recordIdSize, int chainH Buffer.MemoryCopy(thisChainPointer, otherChainPointer, len, len); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal byte* GetRecordIdPtr() + => (byte*)Unsafe.AsPointer(ref this); + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal long* GetChainLinkPtrs(int recordIdSize) => (long*)((byte*)Unsafe.AsPointer(ref this) + recordIdSize); @@ -62,6 +66,12 @@ internal ChainPost(int chainHeight, int recordIdSize) public long* GetChainLinkPtrs(ref PSFValue value) => value.GetChainLinkPtrs(this.RecordIdSize); + + public unsafe void SetRecordId(ref PSFValue value, TRecId recordId) + { + var recIdPtr = value.GetRecordIdPtr(); + Buffer.MemoryCopy(Unsafe.AsPointer(ref recordId), recIdPtr, this.RecordIdSize, this.RecordIdSize); + } } } } From 71377370e55729758037159218f5d1a75733c3da Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Sat, 6 Jun 2020 00:24:18 -0700 Subject: [PATCH 03/19] Clear out a bunch of TODOs: - Use PSF*Exception (derived from FasterException) - Do argument validation: non-null; unique PSF names; ensure PSF belongs to the FasterKV, etc. - Minor optimization for tombstoned records that end PSF chains - Fix PSFRegistrationSettings initialization and validation - Fix up serialNum --- .../FasterPSFSample/BlittableOrders.cs | 13 +-- cs/playground/FasterPSFSample/FPSF.cs | 39 +++++++-- .../FasterPSFSample/FasterPSFSample.cs | 3 +- cs/playground/FasterPSFSample/IOrders.cs | 4 +- cs/playground/FasterPSFSample/Key.cs | 2 + cs/playground/FasterPSFSample/LogFiles.cs | 28 ++++++- cs/playground/FasterPSFSample/ObjectOrders.cs | 15 +--- .../core/Index/PSF/FasterKVPSFDefinition.cs | 2 +- .../Index/PSF/FasterPSFContextOperations.cs | 2 +- cs/src/core/Index/PSF/FasterPSFImpl.cs | 25 +++--- cs/src/core/Index/PSF/IPSF.cs | 2 +- cs/src/core/Index/PSF/PSF.cs | 8 +- cs/src/core/Index/PSF/PSFCompositeKey.cs | 4 +- cs/src/core/Index/PSF/PSFException.cs | 56 +++++++++++++ cs/src/core/Index/PSF/PSFFunctions.cs | 19 ++--- cs/src/core/Index/PSF/PSFGroup.cs | 21 +++-- cs/src/core/Index/PSF/PSFInput.cs | 22 +++-- cs/src/core/Index/PSF/PSFManager.cs | 83 ++++++++++++++++--- 18 files changed, 250 insertions(+), 98 deletions(-) create mode 100644 cs/src/core/Index/PSF/PSFException.cs diff --git a/cs/playground/FasterPSFSample/BlittableOrders.cs b/cs/playground/FasterPSFSample/BlittableOrders.cs index 0ac303faa..b08e1d056 100644 --- a/cs/playground/FasterPSFSample/BlittableOrders.cs +++ b/cs/playground/FasterPSFSample/BlittableOrders.cs @@ -9,6 +9,8 @@ namespace FasterPSFSample { public struct BlittableOrders : IOrders { + public int Id { get; set; } + // Colors, strings, and enums are not blittable so we use int public int SizeInt { get; set; } @@ -16,15 +18,6 @@ public struct BlittableOrders : IOrders public int NumSold { get; set; } - public BlittableOrders(Constants.Size size, Color color, int numSold) - { - this.SizeInt = (int)size; - this.ColorArgb = color.ToArgb(); - this.NumSold = numSold; - } - - public (int, int, int) MemberTuple => (this.SizeInt, this.ColorArgb, this.NumSold); - public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {NumSold}"; public class Functions : IFunctions, Context> @@ -38,7 +31,7 @@ public void SingleReader(ref Key key, ref Input input, ref BlittableOrders value public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) { - if (output.Value.MemberTuple != FasterPSFSample.keyDict[key].MemberTuple) + if (((IOrders)output.Value).MemberTuple != key.MemberTuple) throw new Exception("Read mismatch error!"); } #endregion Read diff --git a/cs/playground/FasterPSFSample/FPSF.cs b/cs/playground/FasterPSFSample/FPSF.cs index 77feb0b14..f963952be 100644 --- a/cs/playground/FasterPSFSample/FPSF.cs +++ b/cs/playground/FasterPSFSample/FPSF.cs @@ -22,19 +22,46 @@ class FPSF internal FPSF(bool useObjectValues, bool useReadCache) { - this.logFiles = new LogFiles(useObjectValues, useReadCache); + this.logFiles = new LogFiles(useObjectValues, useReadCache, 2); this.fht = new FasterKV, TFunctions>( 1L << 20, new TFunctions(), this.logFiles.LogSettings, null, // TODO: add checkpoints - useObjectValues - ? new SerializerSettings { valueSerializer = () => new TSerializer() } : null); + useObjectValues ? new SerializerSettings { valueSerializer = () => new TSerializer() } : null); - this.SizePsf = fht.RegisterPSF("sizePsf", (k, v) => new SizeKey((Constants.Size)v.SizeInt)); - this.ColorPsf = fht.RegisterPSF("colorPsf", (k, v) => new ColorKey(Constants.ColorDict[v.ColorArgb])); + var psfOrdinal = 0; + this.SizePsf = fht.RegisterPSF("sizePsf", (k, v) => new SizeKey((Constants.Size)v.SizeInt), + CreatePSFRegistrationSettings(psfOrdinal++)); + this.ColorPsf = fht.RegisterPSF("colorPsf", (k, v) => new ColorKey(Constants.ColorDict[v.ColorArgb]), + CreatePSFRegistrationSettings(psfOrdinal++)); } - internal void Close() + PSFRegistrationSettings CreatePSFRegistrationSettings(int psfOrdinal) + { + var regSettings = new PSFRegistrationSettings + { + HashTableSize = 1L << 20, + LogSettings = this.logFiles.PSFLogSettings[psfOrdinal], + CheckpointSettings = null, // TODO checkpoints + IPU1CacheSize = 0, // TODO IPUCache + IPU2CacheSize = 0 + }; + + // Override some things. + var regLogSettings = regSettings.LogSettings; + regLogSettings.PageSizeBits = 20; + regLogSettings.SegmentSizeBits = 25; + regLogSettings.MemorySizeBits = 29; + regLogSettings.CopyReadsToTail = false; // TODO--test this in both primary and secondary FKV + if (!(regLogSettings.ReadCacheSettings is null)) + { + regLogSettings.ReadCacheSettings.PageSizeBits = regLogSettings.PageSizeBits; + regLogSettings.ReadCacheSettings.MemorySizeBits = regLogSettings.MemorySizeBits; + } + return regSettings; + } + + internal void Close() { if (!(this.fht is null)) { diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs index fd645e7c6..b6b249c7f 100644 --- a/cs/playground/FasterPSFSample/FasterPSFSample.cs +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -83,6 +83,7 @@ internal static void RunUpserts(FPSF(FPSF (this.SizeInt, this.ColorArgb, this.NumSold); + (int, int, int, int) MemberTuple => (this.Id, this.SizeInt, this.ColorArgb, this.NumSold); } } diff --git a/cs/playground/FasterPSFSample/Key.cs b/cs/playground/FasterPSFSample/Key.cs index 9b873ca95..77025b6ae 100644 --- a/cs/playground/FasterPSFSample/Key.cs +++ b/cs/playground/FasterPSFSample/Key.cs @@ -16,5 +16,7 @@ public struct Key : IFasterEqualityComparer public long GetHashCode64(ref Key key) => Utility.GetHashCode(key.Id); public bool Equals(ref Key k1, ref Key k2) => k1.Id == k2.Id; + + public (int, int, int, int) MemberTuple => FasterPSFSample.keyDict[this].MemberTuple; } } diff --git a/cs/playground/FasterPSFSample/LogFiles.cs b/cs/playground/FasterPSFSample/LogFiles.cs index 1fc84a855..b9cab54d1 100644 --- a/cs/playground/FasterPSFSample/LogFiles.cs +++ b/cs/playground/FasterPSFSample/LogFiles.cs @@ -7,21 +7,37 @@ class LogFiles { private IDevice log; private IDevice objLog; + private IDevice[] PSFDevices; internal LogSettings LogSettings { get; } - internal LogFiles(bool useObjectValue, bool useReadCache) + internal LogSettings[] PSFLogSettings { get; } + + internal string LogDir; + + internal LogFiles(bool useObjectValue, bool useReadCache, int numPSFGroups) { + this.LogDir = Path.Combine(Path.GetTempPath(), "FasterPSFSample"); + // Create files for storing data. We only use one write thread to avoid disk contention. // We set deleteOnClose to true, so logs will auto-delete on completion. - this.log = Devices.CreateLogDevice(Path.GetTempPath() + "hlog.log", deleteOnClose: true); + this.log = Devices.CreateLogDevice(Path.Combine(this.LogDir, "hlog.log"), deleteOnClose: true); if (useObjectValue) - this.objLog = Devices.CreateLogDevice(Path.GetTempPath() + "hlog.obj.log", deleteOnClose: true); + this.objLog = Devices.CreateLogDevice(Path.Combine(this.LogDir, "hlog.obj.log"), deleteOnClose: true); - // Define settings for log this.LogSettings = new LogSettings { LogDevice = log, ObjectLogDevice = objLog }; if (useReadCache) this.LogSettings.ReadCacheSettings = new ReadCacheSettings(); + + this.PSFDevices = new IDevice[numPSFGroups]; + this.PSFLogSettings = new LogSettings[numPSFGroups]; + for (var ii = 0; ii < numPSFGroups; ++ii) + { + this.PSFDevices[ii] = Devices.CreateLogDevice(Path.Combine(this.LogDir, $"psfgroup_{ii}.hlog.log"), deleteOnClose: true); + this.PSFLogSettings[ii] = new LogSettings { LogDevice = log }; + if (useReadCache) + this.PSFLogSettings[ii].ReadCacheSettings = new ReadCacheSettings(); + } } internal void Close() @@ -36,6 +52,10 @@ internal void Close() this.objLog.Close(); this.objLog = null; } + + foreach (var psfDevice in this.PSFDevices) + psfDevice.Close(); + this.PSFDevices = null; } } } diff --git a/cs/playground/FasterPSFSample/ObjectOrders.cs b/cs/playground/FasterPSFSample/ObjectOrders.cs index af84a5ddd..17011cae4 100644 --- a/cs/playground/FasterPSFSample/ObjectOrders.cs +++ b/cs/playground/FasterPSFSample/ObjectOrders.cs @@ -3,12 +3,14 @@ using FASTER.core; using System; -using System.Drawing; namespace FasterPSFSample { public class ObjectOrders : IOrders { + public int Id { get; set; } + + // Colors, strings, and enums are not blittable so we use int public int SizeInt { get => values[0]; set => values[0] = value; } public int ColorArgb { get => values[1]; set => values[1] = value; } @@ -19,15 +21,6 @@ public class ObjectOrders : IOrders public ObjectOrders() => throw new InvalidOperationException("Must use ctor overload"); - public ObjectOrders(Constants.Size size, Color color, int numSold) - { - this.SizeInt = (int)size; - this.ColorArgb = color.ToArgb(); - this.NumSold = numSold; - } - - public (int, int, int) MemberTuple => (this.SizeInt, this.ColorArgb, this.NumSold); - public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {NumSold}"; public class Serializer : BinaryObjectSerializer @@ -57,7 +50,7 @@ public void SingleReader(ref Key key, ref Input input, ref ObjectOrders value, r public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) { - if (output.Value.MemberTuple != FasterPSFSample.keyDict[key].MemberTuple) + if (((IOrders)output.Value).MemberTuple != key.MemberTuple) throw new Exception("Read mismatch error!"); } #endregion Read diff --git a/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs b/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs index 2f8cb1aca..6b2ce1b4e 100644 --- a/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs +++ b/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs @@ -41,7 +41,7 @@ public class FasterKVPSFDefinition : IPSFDefinition Predicate(ref record.GetKey(), ref record.GetValue()); /// - /// The Name of the PSF, assigned by the caller. Must be unique among all PSFs (TODO: enforce). + /// The Name of the PSF, assigned by the caller. Must be unique among all PSFs. /// public string Name { get; } diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs index 0da96cd3c..f7c6b5cbe 100644 --- a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -91,7 +91,7 @@ private Status PsfRcuInsert(GroupKeys groupKeys, ref Value value, ref Input inpu unsafe { psfInput.SetFlags(groupKeys.ResultFlags); } psfInput.IsDelete = false; var internalStatus = this.PsfInternalInsert(ref groupKeys.GetCompositeKeyRef(), ref value, ref input, - ref pcontext, sessionCtx, serialNo /* todo */); + ref pcontext, sessionCtx, serialNo); return internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND ? (Status)internalStatus : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index b1e8ac705..a5b86816e 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -169,8 +169,6 @@ internal OperationStatus PsfInternalReadAddress( var psfInput = psfArgs.Input; var psfOutput = psfArgs.Output; - // TODO: For the primary FasterKV, we do not have any queryKey here to get hash and tag and do latching; - // verify this wrt RelaxedCRT // TODO: Verify we should always find the LogicalAddress OperationStatus status; @@ -224,7 +222,7 @@ ref hlog.GetValue(physicalAddress), else if (logicalAddress >= hlog.BeginAddress) { status = OperationStatus.RECORD_ON_DISK; -#if false // TODO: See discussion for this in Teams (and email) +#if false // TODO: See above and discussion in Teams/email; need to get the key here so we can lock the hash etc. if (sessionCtx.phase == Phase.PREPARE) { Debug.Assert(heldOperation != LatchOperation.Exclusive); @@ -251,7 +249,7 @@ ref hlog.GetValue(physicalAddress), #endregion -#region Create pending context + #region Create pending context CreatePendingContext: { pendingContext.type = OperationType.PSF_READ_ADDRESS; @@ -289,7 +287,7 @@ internal OperationStatus PsfInternalInsert( // Update the PSFValue links for chains with IsNullAt false (indicating a match with the // corresponding PSF) to point to the previous records for all keys in the composite key. - // TODO: We're not checking for a previous occurrence of the PSFValue's recordId because + // Note: We're not checking for a previous occurrence of the PSFValue's recordId because // we are doing insert only here; the update part of upsert is done in PsfInternalUpdate. var chainHeight = this.chainPost.ChainHeight; long* hashes = stackalloc long[chainHeight]; @@ -319,7 +317,7 @@ internal OperationStatus PsfInternalInsert( { logicalAddress = entry.Address; - // TODO: If this happens for any of the TPSFKeys in the composite key, we'll create the pending + // TODO: Test this: If this fails for any TPSFKey in the composite key, we'll create the pending // context and come back here on the retry and overwrite any previously-obtained logicalAddress // at the chainLinkPtr. if (UseReadCache && ReadFromCache(ref compositeKey, ref logicalAddress, ref physicalAddress, @@ -350,11 +348,16 @@ internal OperationStatus PsfInternalInsert( } } - // The chain might extend past the tombstoned record so we must include it in the chain. - // TODO: check to see if the tombstoned record's chain link for this psf is kInvalidAddress - // and disinclude it if so. - //if (!hlog.GetInfo(physicalAddress).Tombstone) - *chainLinkPtr = logicalAddress; + if (hlog.GetInfo(physicalAddress).Tombstone) + { + // The chain might extend past a tombstoned record so we must include it in the chain + // unless its link at chainLinkIdx is kInvalidAddress. + long* prevLinks = this.chainPost.GetChainLinkPtrs(ref hlog.GetValue(physicalAddress)); + long* prevLink = prevLinks + chainLinkIdx; + if (*prevLink == Constants.kInvalidAddress) + continue; + } + *chainLinkPtr = logicalAddress; } #endregion } diff --git a/cs/src/core/Index/PSF/IPSF.cs b/cs/src/core/Index/PSF/IPSF.cs index fc9f59a29..5d03bf0d2 100644 --- a/cs/src/core/Index/PSF/IPSF.cs +++ b/cs/src/core/Index/PSF/IPSF.cs @@ -9,7 +9,7 @@ namespace FASTER.core public interface IPSF { /// - /// The name of the ; must be unique among all // TODO ensure this + /// The name of the ; must be unique among all /// s. /// string Name { get; } diff --git a/cs/src/core/Index/PSF/PSF.cs b/cs/src/core/Index/PSF/PSF.cs index 9f3e79025..b4b420d77 100644 --- a/cs/src/core/Index/PSF/PSF.cs +++ b/cs/src/core/Index/PSF/PSF.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. +using System; using System.Collections.Generic; namespace FASTER.core @@ -16,10 +17,14 @@ public class PSF : IPSF { private readonly IQueryPSF psfGroup; - internal long GroupId { get; } // unique in the PSFManager.psfGroup list + internal long GroupId { get; } // unique in the PSFManager.psfGroup list internal int PsfOrdinal { get; } // in the psfGroup + // PSFs are passed by the caller to the session QueryPSF functions, so make sure they don't send + // a PSF from a different FKV. + internal Guid Id { get; } + /// public string Name { get; } @@ -29,6 +34,7 @@ internal PSF(long groupId, int psfOrdinal, string name, IQueryPSF diff --git a/cs/src/core/Index/PSF/PSFCompositeKey.cs b/cs/src/core/Index/PSF/PSFCompositeKey.cs index d7394e7d0..01e0f3992 100644 --- a/cs/src/core/Index/PSF/PSFCompositeKey.cs +++ b/cs/src/core/Index/PSF/PSFCompositeKey.cs @@ -60,10 +60,10 @@ internal class VarLenLength : IVariableLengthStruct> internal class UnusedKeyComparer : IFasterEqualityComparer> { public long GetHashCode64(ref PSFCompositeKey cKey) - => throw new InvalidOperationException("Must use the overload with PSF ordinal"); + => throw new PSFInvalidOperationException("Must use the overload with PSF ordinal"); public bool Equals(ref PSFCompositeKey cKey1, ref PSFCompositeKey cKey2) - => throw new InvalidOperationException("Must use the overload with PSF ordinal"); + => throw new PSFInvalidOperationException("Must use the overload with PSF ordinal"); } internal unsafe struct PtrWrapper : IDisposable { diff --git a/cs/src/core/Index/PSF/PSFException.cs b/cs/src/core/Index/PSF/PSFException.cs new file mode 100644 index 000000000..7c10023af --- /dev/null +++ b/cs/src/core/Index/PSF/PSFException.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.Serialization; + +namespace FASTER.core +{ + /// + /// FASTER PSF exception base type + /// + public class PSFException : FasterException + { + internal PSFException() { } + + internal PSFException(string message) : base(message) { } + + internal PSFException(string message, Exception innerException) : base(message, innerException) { } + } + + /// + /// FASTER PSF argument exception type + /// + public class PSFArgumentException : PSFException + { + internal PSFArgumentException() { } + + internal PSFArgumentException(string message) : base(message) { } + + internal PSFArgumentException(string message, Exception innerException) : base(message, innerException) { } + } + + /// + /// FASTER PSF argument exception type + /// + public class PSFInvalidOperationException : PSFException + { + internal PSFInvalidOperationException() { } + + internal PSFInvalidOperationException(string message) : base(message) { } + + internal PSFInvalidOperationException(string message, Exception innerException) : base(message, innerException) { } + } + + /// + /// FASTER PSF argument exception type + /// + public class PSFInternalErrorException : PSFException + { + internal PSFInternalErrorException() { } + + internal PSFInternalErrorException(string message) : base($"Internal Error: {message}") { } + + internal PSFInternalErrorException(string message, Exception innerException) : base($"Internal Error: {message}", innerException) { } + } +} diff --git a/cs/src/core/Index/PSF/PSFFunctions.cs b/cs/src/core/Index/PSF/PSFFunctions.cs index 5c16e9aa7..a93db72d8 100644 --- a/cs/src/core/Index/PSF/PSFFunctions.cs +++ b/cs/src/core/Index/PSF/PSFFunctions.cs @@ -19,7 +19,7 @@ public class PSFFunctions : IFunctions> chainPost; - // TODO: remove stuff that has been moved to PSFOutput, etc. + // TODO: remove stuff that has been moved to PSFOutput.Visit, etc. internal PSFFunctions(IChainPost> chainPost) => this.chainPost = chainPost; @@ -40,15 +40,10 @@ public void UpsertCompletionCallback(ref PSFCompositeKey _, ref PSFValu #region Reads public void ConcurrentReader(ref PSFCompositeKey key, ref PSFInputSecondary input, ref PSFValue value, ref PSFOutputSecondary dst) - => this.SingleReader(ref key, ref input, ref value, ref dst); + => throw new PSFInternalErrorException("PSFOutput.Visit instead of ConcurrentReader should be called on PSF-implementing FasterKVs"); public unsafe void SingleReader(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue value, ref PSFOutputSecondary dst) - { - /* TODO is SingleReader still used? - dst.RecordId = value.RecordId; - dst.previousChainLinkLogicalAddress = *(this.chainPost.GetChainLinkPtrs(ref value) + input.PsfOrdinal); - */ - } + => throw new PSFInternalErrorException("PSFOutput.Visit instead of SingleReader should be called on PSF-implementing FasterKVs"); public void ReadCompletionCallback(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFOutputSecondary output, PSFContext ctx, Status status) { /* TODO */ } @@ -56,16 +51,16 @@ public void ReadCompletionCallback(ref PSFCompositeKey _, ref PSFInputS #region RMWs public void CopyUpdater(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue oldValue, ref PSFValue newValue) - => newValue = oldValue; // TODO ensure no deepcopy needed + => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); public void InitialUpdater(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue value) - { /* TODO */ } + => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); public bool InPlaceUpdater(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue value) - { return true; /* TODO */ } + => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); public void RMWCompletionCallback(ref PSFCompositeKey _, ref PSFInputSecondary input, PSFContext ctx, Status status) - { /* TODO */ } + => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); #endregion RMWs public void DeleteCompletionCallback(ref PSFCompositeKey _, PSFContext ctx) diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index 4982c9a86..e8139cef1 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -85,8 +85,6 @@ PSFOutputSecondary, PSFContext, PSFFunctionsOptional registration settings public PSFGroup(long id, IPSFDefinition[] defs, PSFRegistrationSettings regSettings) { - // TODO check for null defs regSettings etc; for PSFs defined on a FasterKV instance we create intelligent - // defaults in regSettings but other clients will have to specify at least hashTableSize, logSettings, etc. this.psfDefinitions = defs; this.Id = id; this.regSettings = regSettings; @@ -156,7 +154,7 @@ private IFasterEqualityComparer GetUserKeyComparer() /// public PSF this[string name] => Array.Find(this.PSFs, psf => psf.Name.Equals(name, StringComparison.CurrentCultureIgnoreCase)) - ?? throw new InvalidOperationException("TODO PSF Exception classes: PSF not found"); + ?? throw new PSFArgumentException("PSF not found"); private unsafe void StoreKeys(ref GroupKeys keys, byte* kPtr, int kLen, PSFResultFlags* flagsPtr, int flagsLen) { @@ -205,6 +203,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor for (int ii = 0; ii < this.psfDefinitions.Length; ++ii) { var key = this.psfDefinitions[ii].Execute(providerData); + *(flags + ii) = key.HasValue ? PSFResultFlags.None : PSFResultFlags.IsNull; if (key.HasValue) { @@ -263,14 +262,14 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor var session = this.GetSession(); try { + var lsn = session.ctx.serialNum + 1; return phase switch { - PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey, ref psfValue, ref input, 1 /*TODO: lsn*/), - PSFExecutePhase.PostUpdate => session.PsfUpdate(ref changeTracker.GetGroupRef(groupOrdinal), ref psfValue, ref input, 1 /*TODO: lsn*/, - changeTracker), - PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey, ref psfValue, ref input, 1 /*TODO: lsn*/, - changeTracker), - _ => throw new FasterException("TODO: Make this a FasterInvalidOperationException") + PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey, ref psfValue, ref input, lsn), + PSFExecutePhase.PostUpdate => session.PsfUpdate(ref changeTracker.GetGroupRef(groupOrdinal), + ref psfValue, ref input, lsn, changeTracker), + PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey, ref psfValue, ref input, lsn, changeTracker), + _ => throw new PSFInternalErrorException("Unknown PSF execution Phase {phase}") }; } finally @@ -361,7 +360,7 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe HashSet deadRecs = null; try { - status = session.PsfReadKey(ref queryKeyPtr.GetRef(), ref readArgs, 1 /*TODO lsn*/); + status = session.PsfReadKey(ref queryKeyPtr.GetRef(), ref readArgs, session.ctx.serialNum + 1); if (status != Status.OK) // TODO check other status yield break; @@ -380,7 +379,7 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe do { readArgs.Input.ReadLogicalAddress = secondaryOutput.PreviousLogicalAddress; - status = session.PsfReadAddress(ref readArgs, 1 /*TODO lsn*/); + status = session.PsfReadAddress(ref readArgs, session.ctx.serialNum + 1); if (status != Status.OK) // TODO check other status yield break; diff --git a/cs/src/core/Index/PSF/PSFInput.cs b/cs/src/core/Index/PSF/PSFInput.cs index d32de2197..e52d95e2c 100644 --- a/cs/src/core/Index/PSF/PSFInput.cs +++ b/cs/src/core/Index/PSF/PSFInput.cs @@ -13,7 +13,8 @@ namespace FASTER.core /// /// The type of the Key, either a for the /// secondary FasterKV instances, or the user's TKVKey for the primary FasterKV instance. - /// The interface separation is needed for the PendingContext + /// The interface separation is needed for the PendingContext, and for the "TPSFKey : struct" + /// constraint in PSFInputSecondary public interface IPSFInput { unsafe void SetFlags(PSFResultFlags* resultFlags); @@ -73,37 +74,34 @@ internal PSFInputPrimaryReadAddress(long readLA) this.ReadLogicalAddress = readLA; } - // TODO: Trim this class and IPSFInput down to just ReadLogicalAddress.. casting to SecondaryInput - // in PsfInternalInsert should be sufficient - /// - public long GroupId => throw new InvalidOperationException("Not valid for Primary FKV"); + public long GroupId => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); /// public int PsfOrdinal { get => Constants.kInvalidPsfOrdinal; - set => throw new InvalidOperationException("Not valid for Primary FKV"); + set => throw new PSFInvalidOperationException("Not valid for Primary FasterFKV"); } public void SetFlags(PSFResultFlags* resultFlags) - => throw new InvalidOperationException("Not valid for Primary FKV"); + => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); - public bool IsNullAt => throw new InvalidOperationException("Not valid for Primary FKV"); + public bool IsNullAt => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); public bool IsDelete { - get => throw new InvalidOperationException("Not valid for Primary FKV"); - set => throw new InvalidOperationException("Not valid for Primary FKV"); + get => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); + set => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); } public long ReadLogicalAddress { get; set; } public long GetHashCode64At(ref TKey key) - => throw new InvalidOperationException("Not valid for Primary FKV"); + => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); public bool EqualsAt(ref TKey queryKey, ref TKey storedKey) - => throw new InvalidOperationException("Not valid for Primary FKV"); + => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); } /// diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index d338862ce..1cc0769dc 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -2,15 +2,19 @@ // Licensed under the MIT license. using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; using System.Threading; namespace FASTER.core { internal class PSFManager where TRecordId : struct, IComparable { - readonly Dictionary> psfGroups - = new Dictionary>(); + private readonly ConcurrentDictionary> psfGroups + = new ConcurrentDictionary>(); + + private readonly ConcurrentDictionary psfNames = new ConcurrentDictionary(); internal bool HasPSFs => this.psfGroups.Count > 0; @@ -79,27 +83,79 @@ internal PSFChangeTracker CreateChangeTracker() private void AddGroup(PSFGroup group) where TPSFKey : struct { var gId = Interlocked.Increment(ref NextGroupId); - this.psfGroups.Add(gId - 1, group); + this.psfGroups.TryAdd(gId - 1, group); + } + + private void VerifyIsBlittable() + { + if (!Utility.IsBlittable()) + throw new PSFArgumentException("The PSF Key type must be blittable."); + } + + private void VerifyIsOurPSF(PSF psf) + { + if (!this.psfNames.TryGetValue(psf.Name, out Guid id) || id != psf.Id) + throw new PSFArgumentException($"The PSF {psf.Name} is not registered with this FasterKV."); } internal PSF RegisterPSF(IPSFDefinition def, PSFRegistrationSettings registrationSettings) where TPSFKey : struct { - // TODO: Runtime check that TPSFKey is blittable - var group = new PSFGroup(this.psfGroups.Count, new[] { def }, registrationSettings); - AddGroup(group); - return group[def.Name]; + this.VerifyIsBlittable(); + if (def is null) + throw new PSFArgumentException("PSF definition cannot be null"); + + // This is a very rare operation and unlikely to have any contention, and locking the dictionary + // makes it much easier to recover from duplicates if needed. + lock (this.psfNames) + { + if (psfNames.ContainsKey(def.Name)) + throw new PSFArgumentException($"A PSF named {def.Name} is already registered in another group"); + var group = new PSFGroup(this.psfGroups.Count, new[] { def }, registrationSettings); + AddGroup(group); + var psf = group[def.Name]; + this.psfNames.TryAdd(psf.Name, psf.Id); + return psf; + } } internal PSF[] RegisterPSF(IPSFDefinition[] defs, PSFRegistrationSettings registrationSettings) where TPSFKey : struct { - // TODO: Runtime check that TPSFKey is blittable - var group = new PSFGroup(this.psfGroups.Count, defs, registrationSettings); - AddGroup(group); - return group.PSFs; + this.VerifyIsBlittable(); + if (defs is null || defs.Length == 0 || defs.Any(def => def is null) || defs.Length == 0) + throw new PSFArgumentException("PSF definitions cannot be null"); + + // For PSFs defined on a FasterKV instance we create intelligent defaults in regSettings. + if (registrationSettings is null) + throw new PSFArgumentException("PSFRegistrationSettings is required"); + if (registrationSettings.LogSettings is null) + throw new PSFArgumentException("PSFRegistrationSettings.LogSettings is required"); + + // This is a very rare operation and unlikely to have any contention, and locking the dictionary + // makes it much easier to recover from duplicates if needed. + lock (this.psfNames) + { + for (var ii = 0; ii < defs.Length; ++ii) + { + var def = defs[ii]; + if (psfNames.ContainsKey(def.Name)) + throw new PSFArgumentException($"A PSF named {def.Name} is already registered in another group"); + for (var jj = ii + 1; jj < defs.Length; ++jj) + { + if (defs[jj].Name == def.Name) + throw new PSFArgumentException($"The PSF name {def.Name} cannot be specfied twice"); + } + } + + var group = new PSFGroup(this.psfGroups.Count, defs, registrationSettings); + AddGroup(group); + foreach (var psf in group.PSFs) + this.psfNames.TryAdd(psf.Name, psf.Id); + return group.PSFs; + } } internal IEnumerable QueryPSF(IPSFCreateProviderData providerDataCreator, @@ -107,7 +163,7 @@ internal IEnumerable QueryPSF(IPSFCreateProviderData QueryPSF(IPSFCreateProviderData over a PQ From 5e9ae40c046c4c46306ab6132a317b6ea4650c30 Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Sat, 6 Jun 2020 02:41:53 -0700 Subject: [PATCH 04/19] Remove the IPSFProviderDataCreator interface --- cs/src/core/ClientSession/ClientSession.cs | 14 ----- cs/src/core/Index/PSF/FasterKVProviderData.cs | 2 +- .../Index/PSF/FasterPSFContextOperations.cs | 6 -- cs/src/core/Index/PSF/FasterPSFImpl.cs | 2 +- .../Index/PSF/FasterPSFSessionOperations.cs | 40 +++++++++--- cs/src/core/Index/PSF/IExecutePSF.cs | 11 ---- .../core/Index/PSF/IPSFCreateProviderData.cs | 16 ----- cs/src/core/Index/PSF/PSF.cs | 2 + cs/src/core/Index/PSF/PSFGroup.cs | 17 ++--- cs/src/core/Index/PSF/PSFManager.cs | 63 ++++++------------- 10 files changed, 63 insertions(+), 110 deletions(-) delete mode 100644 cs/src/core/Index/PSF/IPSFCreateProviderData.cs diff --git a/cs/src/core/ClientSession/ClientSession.cs b/cs/src/core/ClientSession/ClientSession.cs index 5dc5fd594..ed546a007 100644 --- a/cs/src/core/ClientSession/ClientSession.cs +++ b/cs/src/core/ClientSession/ClientSession.cs @@ -21,7 +21,6 @@ namespace FASTER.core /// /// public sealed partial class ClientSession : - IPSFCreateProviderData>, IDisposable where Key : new() where Value : new() @@ -51,19 +50,6 @@ internal ClientSession( /// public string ID { get { return ctx.guid; } } - /// - public FasterKVProviderData Create(long logicalAddress) - { - // Looks up logicalAddress in the primary FasterKV - var psfArgs = new PSFReadArgs(new PSFInputPrimaryReadAddress(logicalAddress), - new PSFOutputPrimaryReadAddress(this.fht.hlog)); - - // Call this directly here, because we are TODO - var status = fht.ContextPsfReadAddress(ref psfArgs, 1 /*TODO lsn*/, ctx); - var primaryOutput = psfArgs.Output as IPSFPrimaryOutput>; - return status == Status.OK ? primaryOutput.ProviderData : null; // TODO check other states - } - /// /// Dispose session /// diff --git a/cs/src/core/Index/PSF/FasterKVProviderData.cs b/cs/src/core/Index/PSF/FasterKVProviderData.cs index 40e064ec5..9a9166fd5 100644 --- a/cs/src/core/Index/PSF/FasterKVProviderData.cs +++ b/cs/src/core/Index/PSF/FasterKVProviderData.cs @@ -21,7 +21,7 @@ public class FasterKVProviderData : IDisposable { // C# doesn't allow ref fields and even if it did, if the client held the FasterKVProviderData // past the ref lifetime, bad things would happen when accessing the ref key/value. - internal IHeapContainer keyContainer; // TODO: Perf efficiency + internal IHeapContainer keyContainer; internal IHeapContainer valueContainer; internal FasterKVProviderData(AllocatorBase allocator, ref TKVKey key, ref TKVValue value) diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs index f7c6b5cbe..fafa5bb40 100644 --- a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -105,12 +105,6 @@ internal Status ContextPsfDelete(ref Key key, ref Valu { var pcontext = default(PendingContext); - // TODO check cache first: - // var internalStatus = this.PsfInternalDeleteByCache(ref key, ref input, - // ref pcontext, sessionCtx, serialNo)); - // if (internalStatus == OperationStatus.NOTFOUND) - // ... do below ... - var psfInput = (IPSFInput)input; psfInput.IsDelete = true; var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index a5b86816e..01c9013a6 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -222,7 +222,7 @@ ref hlog.GetValue(physicalAddress), else if (logicalAddress >= hlog.BeginAddress) { status = OperationStatus.RECORD_ON_DISK; -#if false // TODO: See above and discussion in Teams/email; need to get the key here so we can lock the hash etc. +#if false // TODO: See discussion in Teams/email; need to get the key here so we can get the hash, latch, etc. if (sessionCtx.phase == Phase.PREPARE) { Debug.Assert(heldOperation != LatchOperation.Exclusive); diff --git a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs index 172a26c8c..b54316ba4 100644 --- a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs @@ -3,11 +3,11 @@ using System; using System.Collections.Generic; +using System.Linq; namespace FASTER.core { public sealed partial class ClientSession : - IPSFCreateProviderData>, IDisposable where Key : new() where Value : new() @@ -91,6 +91,29 @@ internal Status PsfDelete(ref Key key, ref Value value #endregion PSF calls for Secondary FasterKV #region PSF Query API for primary FasterKV + + internal FasterKVProviderData CreateProviderData(long logicalAddress) + { + // Looks up logicalAddress in the primary FasterKV + var psfArgs = new PSFReadArgs(new PSFInputPrimaryReadAddress(logicalAddress), + new PSFOutputPrimaryReadAddress(this.fht.hlog)); + + // Call this directly here because we are within UnsafeResumeThread. TODO: is that OK with an Iterator? + var status = fht.ContextPsfReadAddress(ref psfArgs, this.ctx.serialNum + 1, ctx); + var primaryOutput = psfArgs.Output as IPSFPrimaryOutput>; + return status == Status.OK ? primaryOutput.ProviderData : null; // TODO check other status + } + + internal IEnumerable> ReturnProviderDatas(IEnumerable logicalAddresses) + { + foreach (var logicalAddress in logicalAddresses) + { + var providerData = this.CreateProviderData(logicalAddress); + if (!(providerData is null)) + yield return providerData; + } + } + /// /// Issue a query on a single on a single key value. /// @@ -111,7 +134,7 @@ public IEnumerable> QueryPSF( if (SupportAsync) UnsafeResumeThread(); try { - return this.fht.PSFManager.QueryPSF(this, psf, psfKey, querySettings); + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, psfKey, querySettings)); } finally { @@ -140,7 +163,7 @@ public IEnumerable> QueryPSF if (SupportAsync) UnsafeResumeThread(); try { - return this.fht.PSFManager.QueryPSF(this, psf, psfKeys, querySettings); + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, psfKeys, querySettings)); } finally { @@ -179,7 +202,8 @@ public IEnumerable> QueryPSF(this, psf1, psfKey1, psf2, psfKey2, matchPredicate, querySettings); + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, psfKey1, psf2, psfKey2, + matchPredicate, querySettings)); } finally { @@ -221,7 +245,8 @@ public IEnumerable> QueryPSF(this, psf1, psfKeys1, psf2, psfKeys2, matchPredicate, querySettings); + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, psfKeys1, psf2, psfKeys2, + matchPredicate, querySettings)); } finally { @@ -259,7 +284,7 @@ public IEnumerable> QueryPSF( if (SupportAsync) UnsafeResumeThread(); try { - return this.fht.PSFManager.QueryPSF(this, psfsAndKeys, matchPredicate, querySettings); + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psfsAndKeys, matchPredicate, querySettings)); } finally { @@ -306,7 +331,8 @@ public IEnumerable> QueryPSF Status ExecuteAndStore(TProviderData data, TRecordId recordId, PSFExecutePhase phase, PSFChangeTracker changeTracker); - /// - /// For the given , verify that the - /// matches (returns non-null). - /// - /// The provider data wrapped around the TRecordId - /// The ordinal of the in the - /// . - /// True if the providerData matches (returns non-null from) the , - /// else false. - bool Verify(TProviderData providerData, int psfOrdinal); - /// /// The identifier of this . /// diff --git a/cs/src/core/Index/PSF/IPSFCreateProviderData.cs b/cs/src/core/Index/PSF/IPSFCreateProviderData.cs deleted file mode 100644 index aecf4e275..000000000 --- a/cs/src/core/Index/PSF/IPSFCreateProviderData.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace FASTER.core -{ - /// - /// The data provider's instantiator that takes the RecordId and creates the data class the user sees. - /// - public interface IPSFCreateProviderData - { - /// - /// Creates the provider data from the record id. - /// - TProviderData Create(TRecordId recordId); - } -} diff --git a/cs/src/core/Index/PSF/PSF.cs b/cs/src/core/Index/PSF/PSF.cs index b4b420d77..567304b52 100644 --- a/cs/src/core/Index/PSF/PSF.cs +++ b/cs/src/core/Index/PSF/PSF.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; namespace FASTER.core { @@ -41,6 +42,7 @@ internal PSF(long groupId, int psfOrdinal, string name, IQueryPSFs. /// /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public IEnumerable Query(TPSFKey key) => this.psfGroup.Query(this.PsfOrdinal, key); } diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index e8139cef1..2ddd83e0f 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -281,7 +281,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor /// public Status GetBeforeKeys(PSFChangeTracker changeTracker) { - // Obtain the "before" values. TODO try to find TRecordId in the IPUCache first. + // Obtain the "before" values. TODOcache: try to find TRecordId in the IPUCache first. return ExecuteAndStore(changeTracker.BeforeData, default, PSFExecutePhase.PreUpdate, changeTracker); } @@ -290,16 +290,16 @@ public Status GetBeforeKeys(PSFChangeTracker changeTra /// public Status Update(PSFChangeTracker changeTracker) { - changeTracker.CachedBeforeLA = Constants.kInvalidAddress; // TODO Find BeforeRecordId in IPUCache + changeTracker.CachedBeforeLA = Constants.kInvalidAddress; // TODOcache: Find BeforeRecordId in IPUCache if (changeTracker.CachedBeforeLA != Constants.kInvalidAddress) { if (changeTracker.UpdateOp == UpdateOperation.RCU) { - // TODO: Tombstone it, and possibly unlink; or just copy its keys into changeTracker.Before. + // TODOcache: Tombstone it, and possibly unlink; or just copy its keys into changeTracker.Before. } else { - // TODO: Try to splice in-place; or just copy its keys into changeTracker.Before. + // TODOcache: Try to splice in-place; or just copy its keys into changeTracker.Before. } } else @@ -317,20 +317,15 @@ public Status Update(PSFChangeTracker changeTracker) /// public Status Delete(PSFChangeTracker changeTracker) { - changeTracker.CachedBeforeLA = Constants.kInvalidAddress; // TODO Find BeforeRecordId in IPUCache + changeTracker.CachedBeforeLA = Constants.kInvalidAddress; // TODOcache: Find BeforeRecordId in IPUCache if (changeTracker.CachedBeforeLA != Constants.kInvalidAddress) { - // TODO: Tombstone it, and possibly unlink; or just copy its keys into changeTracker.Before. + // TODOcache: Tombstone it, and possibly unlink; or just copy its keys into changeTracker.Before. // If the latter, we can bypass ExecuteAndStore's PSF-execute loop } return this.ExecuteAndStore(changeTracker.BeforeData, default, PSFExecutePhase.Delete, changeTracker); } - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool Verify(TProviderData providerData, int psfOrdinal) // TODO revise to new spec - => !(this.psfDefinitions[psfOrdinal].Execute(providerData) is null); - /// public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key) { diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index 1cc0769dc..91009fef6 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -21,6 +21,9 @@ private readonly ConcurrentDictionary changeTracker) { + // TODO: RecordId locking, to ensure consistency of multiple PSFs if the same record is updated + // multiple times; possibly a single Array[N] which is locked on TRecordId.GetHashCode % N. + // TODO: This is called from ContextUpsert or InternalCompleteRetryRequest, where it's still // in the Context threading control for the primaryKV. I think it needs to move out of there. @@ -158,13 +161,11 @@ internal PSF[] RegisterPSF(IPSFDefinition QueryPSF(IPSFCreateProviderData providerDataCreator, - PSF psf, - TPSFKey psfKey, PSFQuerySettings querySettings) + internal IEnumerable QueryPSF(PSF psf, + TPSFKey psfKey, PSFQuerySettings querySettings) where TPSFKey : struct { this.VerifyIsOurPSF(psf); - var group = this.psfGroups[psf.GroupId]; foreach (var recordId in psf.Query(psfKey)) { if (querySettings != null && querySettings.CancellationToken.IsCancellationRequested) @@ -173,22 +174,14 @@ internal IEnumerable QueryPSF(IPSFCreateProviderData QueryPSF(IPSFCreateProviderData providerDataCreator, - PSF psf, - TPSFKey[] psfKeys, PSFQuerySettings querySettings) + internal IEnumerable QueryPSF(PSF psf, + TPSFKey[] psfKeys, PSFQuerySettings querySettings) where TPSFKey : struct { - // TODO: Get rid of providerDataCreator this.VerifyIsOurPSF(psf); // TODO implement range queries. This will start retrieval of a stream of returned values for the @@ -197,11 +190,10 @@ internal IEnumerable QueryPSF(IPSFCreateProviderData QueryPSF( - IPSFCreateProviderData providerDataCreator, + internal IEnumerable QueryPSF( PSF psf1, TPSFKey1 psfKey1, PSF psf2, TPSFKey2 psfKey2, Func matchPredicate, @@ -238,12 +230,9 @@ bool cancellationRequested() ? matchPredicate(true, true) : cmp > 0 ? matchPredicate(true, false) : matchPredicate(false, true); if (predResult) - { - // Let the trailing one catch up - var providerData = providerDataCreator.Create(cmp < 0 ? e1.Current : e2.Current); - var verify = cmp <= 0 ? group1.Verify(providerData, psf1.PsfOrdinal) : true - && cmp >= 0 ? group2.Verify(providerData, psf2.PsfOrdinal) : true; - } + yield return cmp < 0 ? e1.Current : e2.Current; + + // Let the trailing one catch up if (cmp <= 0) e1done = !e1.MoveNext(); if (cmp >= 0) @@ -265,48 +254,36 @@ bool cancellationRequested() { var predResult = matchPredicate(!e1done, !e2done); if (predResult) - { - //yield return providerDataCreator(!e1done ? e1.Current : e2.Current); - var providerData = providerDataCreator.Create(!e1done ? e1.Current : e2.Current); - if (!(providerData is null)) - { - var psfOrdinal = !e1done ? psf1.PsfOrdinal : psf2.PsfOrdinal; - if ((!e1done ? group1 : group2).Verify(providerData, psfOrdinal)) - yield return providerData; - } - } + yield return !e1done ? e1.Current : e2.Current; if (!(!e1done ? e1 : e2).MoveNext()) yield break; } } - internal IEnumerable QueryPSF( - IPSFCreateProviderData providerDataCreator, + internal IEnumerable QueryPSF( PSF psf1, TPSFKey1[] psfKeys1, PSF psf2, TPSFKey2[] psfKeys2, Func matchPredicate, PSFQuerySettings querySettings) { // TODO: Similar range-query/PQ implementation (and first-element only execution) as discussed above. - return QueryPSF(providerDataCreator, psf1, psfKeys1[0], psf2, psfKeys2[0], matchPredicate, querySettings); + return QueryPSF(psf1, psfKeys1[0], psf2, psfKeys2[0], matchPredicate, querySettings); } // Power user versions. We could add up to 3. Anything more complicated than // that, they can just post-process with LINQ. - internal IEnumerable QueryPSF( - IPSFCreateProviderData providerDataCreator, + internal IEnumerable QueryPSF( (PSF psf1, TPSFKey[])[] psfsAndKeys, Func matchPredicate, PSFQuerySettings querySettings) { // TODO: Not implemented. The input argument to the predicate is the matches to each // element of psfsAndKeys. - return Array.Empty(); + return Array.Empty(); } - internal IEnumerable QueryPSF( - IPSFCreateProviderData providerDataCreator, + internal IEnumerable QueryPSF( (PSF psf1, TPSFKey1[])[] psfsAndKeys1, (PSF psf2, TPSFKey2[])[] psfsAndKeys2, Func matchPredicate, @@ -315,7 +292,7 @@ internal IEnumerable QueryPSF( // TODO: Not implemented. The first input argument to the predicate is the matches to each // element of psfsAndKeys1; the second input argument to the predicate is the matches to each // element of psfsAndKeys2. - return Array.Empty(); + return Array.Empty(); } } } From e731d46c5b7485d734493fd27e3d758566749f08 Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Sat, 6 Jun 2020 03:10:13 -0700 Subject: [PATCH 05/19] Rename ChainPost->PSFValueAccessor and ChainHeight->PSFCount --- cs/src/core/Index/FASTER/FASTERThread.cs | 2 +- .../Index/PSF/FasterPSFContextOperations.cs | 2 +- cs/src/core/Index/PSF/FasterPSFImpl.cs | 14 +++---- .../{IChainPost.cs => IPSFValueAccessor.cs} | 7 ++-- cs/src/core/Index/PSF/PSFCompositeKey.cs | 14 +++---- cs/src/core/Index/PSF/PSFFunctions.cs | 10 ++--- cs/src/core/Index/PSF/PSFGroup.cs | 37 +++++++++---------- cs/src/core/Index/PSF/PSFOutput.cs | 8 ++-- cs/src/core/Index/PSF/PSFValue.cs | 14 +++---- 9 files changed, 52 insertions(+), 56 deletions(-) rename cs/src/core/Index/PSF/{IChainPost.cs => IPSFValueAccessor.cs} (60%) diff --git a/cs/src/core/Index/FASTER/FASTERThread.cs b/cs/src/core/Index/FASTER/FASTERThread.cs index a0d8b8a41..8e98bd51b 100644 --- a/cs/src/core/Index/FASTER/FASTERThread.cs +++ b/cs/src/core/Index/FASTER/FASTERThread.cs @@ -255,7 +255,7 @@ internal void InternalCompleteRetryRequest(FasterExecutionContext opCtx, FasterE if (pendingContext.psfUpdateArgs.ChangeTracker.FindGroup(psfInput.GroupId, out var ordinal)) { ref GroupKeysPair groupKeysPair = ref pendingContext.psfUpdateArgs.ChangeTracker.GetGroupRef(ordinal); - this.chainPost.SetRecordId(ref value, pendingContext.psfUpdateArgs.ChangeTracker.AfterRecordId); + this.psfValueAccessor.SetRecordId(ref value, pendingContext.psfUpdateArgs.ChangeTracker.AfterRecordId); var pcontext = default(PendingContext); PsfRcuInsert(groupKeysPair.After, ref value, ref pendingContext.input, ref pcontext, currentCtx, pendingContext.serialNum + 1); diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs index fafa5bb40..e13a66ce6 100644 --- a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -78,7 +78,7 @@ internal Status ContextPsfUpdate(ref GroupKeysPair gro if (status == Status.OK) { - this.chainPost.SetRecordId(ref value, changeTracker.AfterRecordId); + this.psfValueAccessor.SetRecordId(ref value, changeTracker.AfterRecordId); return PsfRcuInsert(groupKeysPair.After, ref value, ref input, ref pcontext, sessionCtx, serialNo + 1); } return status; diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index 01c9013a6..b2781ea88 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -17,7 +17,7 @@ public unsafe partial class FasterKV { - internal IChainPost chainPost; + internal IPSFValueAccessor psfValueAccessor; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal OperationStatus PsfInternalReadKey( @@ -289,10 +289,10 @@ internal OperationStatus PsfInternalInsert( // corresponding PSF) to point to the previous records for all keys in the composite key. // Note: We're not checking for a previous occurrence of the PSFValue's recordId because // we are doing insert only here; the update part of upsert is done in PsfInternalUpdate. - var chainHeight = this.chainPost.ChainHeight; - long* hashes = stackalloc long[chainHeight]; - long* chainLinkPtrs = this.chainPost.GetChainLinkPtrs(ref value); - for (var chainLinkIdx = 0; chainLinkIdx < chainHeight; ++chainLinkIdx) + var psfCount = this.psfValueAccessor.PSFCount; + long* hashes = stackalloc long[psfCount]; + long* chainLinkPtrs = this.psfValueAccessor.GetChainLinkPtrs(ref value); + for (var chainLinkIdx = 0; chainLinkIdx < psfCount; ++chainLinkIdx) { // For RCU, or in case we had to retry due to CPR_SHIFT and somehow managed to delete // the previously found record, clear out the chain link pointer. @@ -352,7 +352,7 @@ internal OperationStatus PsfInternalInsert( { // The chain might extend past a tombstoned record so we must include it in the chain // unless its link at chainLinkIdx is kInvalidAddress. - long* prevLinks = this.chainPost.GetChainLinkPtrs(ref hlog.GetValue(physicalAddress)); + long* prevLinks = this.psfValueAccessor.GetChainLinkPtrs(ref hlog.GetValue(physicalAddress)); long* prevLink = prevLinks + chainLinkIdx; if (*prevLink == Constants.kInvalidAddress) continue; @@ -449,7 +449,7 @@ internal OperationStatus PsfInternalInsert( hlog.ShallowCopy(ref compositeKey, ref hlog.GetKey(newPhysicalAddress)); functions.SingleWriter(ref compositeKey, ref value, ref hlog.GetValue(newPhysicalAddress)); - for (var chainLinkIdx = 0; chainLinkIdx < chainHeight; ++chainLinkIdx) + for (var chainLinkIdx = 0; chainLinkIdx < psfCount; ++chainLinkIdx) { psfInput.PsfOrdinal = chainLinkIdx; if (psfInput.IsNullAt) diff --git a/cs/src/core/Index/PSF/IChainPost.cs b/cs/src/core/Index/PSF/IPSFValueAccessor.cs similarity index 60% rename from cs/src/core/Index/PSF/IChainPost.cs rename to cs/src/core/Index/PSF/IPSFValueAccessor.cs index e6fff063e..89c9efabe 100644 --- a/cs/src/core/Index/PSF/IChainPost.cs +++ b/cs/src/core/Index/PSF/IPSFValueAccessor.cs @@ -4,13 +4,12 @@ namespace FASTER.core { /// - /// This represents (a generic decoupling of) the concept that each PSFValue is essentially a "post" - /// that supports the chains, like a post in a barbed-wire fence. + /// This interface is a generic indirection to access the pieces of a PSFValue{TPSFKey}. /// /// - internal unsafe interface IChainPost + internal unsafe interface IPSFValueAccessor { - int ChainHeight { get; } + int PSFCount { get; } int RecordIdSize { get; } diff --git a/cs/src/core/Index/PSF/PSFCompositeKey.cs b/cs/src/core/Index/PSF/PSFCompositeKey.cs index 01e0f3992..49b7aca9b 100644 --- a/cs/src/core/Index/PSF/PSFCompositeKey.cs +++ b/cs/src/core/Index/PSF/PSFCompositeKey.cs @@ -14,11 +14,11 @@ public unsafe struct PSFCompositeKey where TPSFKey : struct { // This class is entirely a "reinterpret_cast" implementation; there are no data members. - internal void CopyTo(ref PSFCompositeKey other, int keySize, int chainHeight) + internal void CopyTo(ref PSFCompositeKey other, int keySize, int psfCount) { var thisKeysPointer = (byte*)Unsafe.AsPointer(ref this); var otherKeysPointer = (byte*)Unsafe.AsPointer(ref other); - var len = keySize * chainHeight; + var len = keySize * psfCount; Buffer.MemoryCopy(thisKeysPointer, otherKeysPointer, len, len); } @@ -48,7 +48,7 @@ internal class VarLenLength : IVariableLengthStruct> { private readonly int size; - internal VarLenLength(int keySize, int chainHeight) => this.size = keySize * chainHeight; + internal VarLenLength(int keySize, int psfCount) => this.size = keySize * psfCount; public int GetAverageLength() => this.size; @@ -88,7 +88,7 @@ internal ref PSFCompositeKey GetRef() internal interface ICompositeKeyComparer { - int ChainHeight { get; } + int PSFCount { get; } long GetHashCode64(int psfOrdinal, ref TCompositeKey cKey); @@ -100,13 +100,13 @@ internal class PSFCompositeKeyComparer : ICompositeKeyComparer userComparer; private readonly int keySize; - public int ChainHeight { get; } + public int PSFCount { get; } - internal PSFCompositeKeyComparer(IFasterEqualityComparer ucmp, int ksize, int chainHeight) + internal PSFCompositeKeyComparer(IFasterEqualityComparer ucmp, int ksize, int psfCount) { this.userComparer = ucmp; this.keySize = ksize; - this.ChainHeight = chainHeight; + this.PSFCount = psfCount; } public long GetHashCode64(int psfOrdinal, ref PSFCompositeKey cKey) diff --git a/cs/src/core/Index/PSF/PSFFunctions.cs b/cs/src/core/Index/PSF/PSFFunctions.cs index a93db72d8..2936a4a4b 100644 --- a/cs/src/core/Index/PSF/PSFFunctions.cs +++ b/cs/src/core/Index/PSF/PSFFunctions.cs @@ -17,22 +17,22 @@ public class PSFFunctions : IFunctions> chainPost; + IPSFValueAccessor> psfValueAccessor; // TODO: remove stuff that has been moved to PSFOutput.Visit, etc. - internal PSFFunctions(IChainPost> chainPost) - => this.chainPost = chainPost; + internal PSFFunctions(IPSFValueAccessor> accessor) + => this.psfValueAccessor = accessor; #region Upserts public bool ConcurrentWriter(ref PSFCompositeKey _, ref PSFValue src, ref PSFValue dst) { - src.CopyTo(ref dst, this.chainPost.RecordIdSize, this.chainPost.ChainHeight); + src.CopyTo(ref dst, this.psfValueAccessor.RecordIdSize, this.psfValueAccessor.PSFCount); return true; } public void SingleWriter(ref PSFCompositeKey _, ref PSFValue src, ref PSFValue dst) - => src.CopyTo(ref dst, this.chainPost.RecordIdSize, this.chainPost.ChainHeight); + => src.CopyTo(ref dst, this.psfValueAccessor.RecordIdSize, this.psfValueAccessor.PSFCount); public void UpsertCompletionCallback(ref PSFCompositeKey _, ref PSFValue value, PSFContext ctx) { /* TODO */ } diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index 2ddd83e0f..b37eab3d5 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -23,7 +23,6 @@ public class PSFGroup : IExecutePSF, PSFValue, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions> fht; private readonly PSFFunctions functions; @@ -57,11 +56,11 @@ PSFOutputSecondary, PSFContext, PSFFunctions public PSF[] PSFs { get; private set; } - private int ChainHeight => this.PSFs.Length; + private int PSFCount => this.PSFs.Length; private readonly IFasterEqualityComparer userKeyComparer; private readonly ICompositeKeyComparer> compositeKeyComparer; - private readonly IChainPost> chainPost; + private readonly IPSFValueAccessor> psfValueAccessor; private readonly SectorAlignedBufferPool bufferPool; @@ -91,25 +90,24 @@ public PSFGroup(long id, IPSFDefinition[] defs, PSFRegis this.userKeyComparer = GetUserKeyComparer(); this.PSFs = defs.Select((def, ord) => new PSF(this.Id, ord, def.Name, this)).ToArray(); - this.compositeKeyComparer = new PSFCompositeKeyComparer(this.userKeyComparer, this.keySize, this.ChainHeight); + this.compositeKeyComparer = new PSFCompositeKeyComparer(this.userKeyComparer, this.keySize, this.PSFCount); - // TODO doc (or change) chainPost terminology - this.chainPost = new PSFValue.ChainPost(this.ChainHeight, this.recordIdSize); + this.psfValueAccessor = new PSFValue.Accessor(this.PSFCount, this.recordIdSize); this.checkpointSettings = regSettings?.CheckpointSettings; - this.functions = new PSFFunctions(chainPost); + this.functions = new PSFFunctions(psfValueAccessor); this.fht = new FasterKV, PSFValue, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions>( - regSettings.HashTableSize, new PSFFunctions(chainPost), + regSettings.HashTableSize, new PSFFunctions(psfValueAccessor), regSettings.LogSettings, this.checkpointSettings, null /*SerializerSettings*/, new PSFCompositeKey.UnusedKeyComparer(), new VariableLengthStructSettings, PSFValue> { - keyLength = new PSFCompositeKey.VarLenLength(this.keySize, this.ChainHeight), - valueLength = new PSFValue.VarLenLength(this.recordIdSize, this.ChainHeight) + keyLength = new PSFCompositeKey.VarLenLength(this.keySize, this.PSFCount), + valueLength = new PSFValue.VarLenLength(this.recordIdSize, this.PSFCount) } ) - { chainPost = chainPost }; + { psfValueAccessor = psfValueAccessor }; this.bufferPool = this.fht.hlog.bufferPool; } @@ -171,7 +169,7 @@ internal unsafe void MarkChanges(GroupKeysPair keysPair) var after = keysPair.After; var beforeCompKey = before.GetCompositeKeyRef>(); var afterCompKey = after.GetCompositeKeyRef>(); - for (var ii = 0; ii < this.ChainHeight; ++ii) + for (var ii = 0; ii < this.PSFCount; ++ii) { bool keysEqual() => beforeCompKey.GetKeyRef(ii, this.keySize).Equals(afterCompKey.GetKeyRef(ii, this.keySize)); @@ -189,16 +187,15 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor { // Note: stackalloc is safe because PendingContext or PSFChangeTracker will copy it to the bufferPool // if needed. On the Insert fast path, we don't want any allocations otherwise; changeTracker is null. - // TODO: Change ChainHeight to psfCount - var keyMemLen = ((keySize * this.ChainHeight + sizeof(int) - 1) & ~(sizeof(int) - 1)); + var keyMemLen = ((keySize * this.PSFCount + sizeof(int) - 1) & ~(sizeof(int) - 1)); var keyMemInt = stackalloc int[keyMemLen / sizeof(int)]; for (var ii = 0; ii < keyMemLen; ++ii) keyMemInt[ii] = 0; var keyMem = (byte*)keyMemInt; ref PSFCompositeKey compositeKey = ref Unsafe.AsRef>(keyMem); - var flagsMemLen = this.ChainHeight * sizeof(PSFResultFlags); - PSFResultFlags* flags = stackalloc PSFResultFlags[this.ChainHeight]; + var flagsMemLen = this.PSFCount * sizeof(PSFResultFlags); + PSFResultFlags* flags = stackalloc PSFResultFlags[this.PSFCount]; var anyMatch = false; for (int ii = 0; ii < this.psfDefinitions.Length; ++ii) { @@ -218,7 +215,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor var input = new PSFInputSecondary(0, compositeKeyComparer, this.Id, flags); - var valueMem = stackalloc byte[recordIdSize + sizeof(long) * this.ChainHeight]; + var valueMem = stackalloc byte[recordIdSize + sizeof(long) * this.PSFCount]; ref PSFValue psfValue = ref Unsafe.AsRef>(valueMem); psfValue.RecordId = recordId; @@ -255,7 +252,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor // We don't need to do anything here for Delete. } - long* chainLinkPtrs = this.fht.chainPost.GetChainLinkPtrs(ref psfValue); + long* chainLinkPtrs = this.fht.psfValueAccessor.GetChainLinkPtrs(ref psfValue); for (int ii = 0; ii < this.psfDefinitions.Length; ++ii) *(chainLinkPtrs + ii) = Constants.kInvalidAddress; @@ -339,7 +336,7 @@ public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key) // TODOperf: PSFInput* and PSFOutput* are classes because we communicate through interfaces to avoid // having to add additional generic args. Interfaces on structs incur boxing overhead (plus the complexity // of mutable structs). But check the performance here; if necessary perhaps I can figure out a way to - // pass a struct with no TRecordId, TPSFKey, etc. and use an FHT-level interface to manage it. + // pass a struct with no TPSFKey, TRecordId, etc. and use an FHT-level interface to manage it. var psfInput = new PSFInputSecondary(psfOrdinal, compositeKeyComparer, this.Id); return Query(keyPtr, psfInput); } @@ -348,7 +345,7 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe PSFInputSecondary input) { var readArgs = new PSFReadArgs, PSFValue>( - input, new PSFOutputSecondary(this.chainPost)); + input, new PSFOutputSecondary(this.psfValueAccessor)); var session = this.GetSession(); Status status; diff --git a/cs/src/core/Index/PSF/PSFOutput.cs b/cs/src/core/Index/PSF/PSFOutput.cs index 1d99a36fc..0d230ade2 100644 --- a/cs/src/core/Index/PSF/PSFOutput.cs +++ b/cs/src/core/Index/PSF/PSFOutput.cs @@ -73,7 +73,7 @@ public unsafe class PSFOutputSecondary : IPSFOutput> chainPost; + IPSFValueAccessor> psfValueAccessor; public TRecordId RecordId { get; private set; } @@ -82,9 +82,9 @@ public unsafe class PSFOutputSecondary : IPSFOutput> chainPost) + internal PSFOutputSecondary(IPSFValueAccessor> accessor) { - this.chainPost = chainPost; + this.psfValueAccessor = accessor; this.RecordId = default; this.PreviousLogicalAddress = Constants.kInvalidAddress; } @@ -95,7 +95,7 @@ public PSFOperationStatus Visit(int psfOrdinal, ref PSFCompositeKey key { this.RecordId = value.RecordId; this.Tombstone = tombstone; - this.PreviousLogicalAddress = *(this.chainPost.GetChainLinkPtrs(ref value) + psfOrdinal); + this.PreviousLogicalAddress = *(this.psfValueAccessor.GetChainLinkPtrs(ref value) + psfOrdinal); return new PSFOperationStatus(OperationStatus.SUCCESS); } } diff --git a/cs/src/core/Index/PSF/PSFValue.cs b/cs/src/core/Index/PSF/PSFValue.cs index ef28dacb2..018805380 100644 --- a/cs/src/core/Index/PSF/PSFValue.cs +++ b/cs/src/core/Index/PSF/PSFValue.cs @@ -19,12 +19,12 @@ public unsafe struct PSFValue /// public TRecordId RecordId; - internal void CopyTo(ref PSFValue other, int recordIdSize, int chainHeight) + internal void CopyTo(ref PSFValue other, int recordIdSize, int psfCount) { other.RecordId = this.RecordId; var thisChainPointer = ((byte*)Unsafe.AsPointer(ref this) + recordIdSize); var otherChainPointer = ((byte*)Unsafe.AsPointer(ref other) + recordIdSize); - var len = sizeof(long) * chainHeight; // The chains links are "long logicalAddress". + var len = sizeof(long) * psfCount; // The chains links are "long logicalAddress". Buffer.MemoryCopy(thisChainPointer, otherChainPointer, len, len); } @@ -43,7 +43,7 @@ internal class VarLenLength : IVariableLengthStruct> { private readonly int size; - internal VarLenLength(int recordIdSize, int chainHeight) => this.size = recordIdSize + sizeof(long) * chainHeight; + internal VarLenLength(int recordIdSize, int psfCount) => this.size = recordIdSize + sizeof(long) * psfCount; public int GetAverageLength() => this.size; @@ -52,15 +52,15 @@ internal class VarLenLength : IVariableLengthStruct> public int GetLength(ref PSFValue _) => this.size; } - internal class ChainPost : IChainPost> + internal class Accessor : IPSFValueAccessor> { - internal ChainPost(int chainHeight, int recordIdSize) + internal Accessor(int psfCount, int recordIdSize) { - this.ChainHeight = chainHeight; + this.PSFCount = psfCount; this.RecordIdSize = recordIdSize; } - public int ChainHeight { get; } + public int PSFCount { get; } public int RecordIdSize { get; } From a432bd1fc7ebb08405f297ef98e4e68c7b653af4 Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Sun, 7 Jun 2020 01:45:53 -0700 Subject: [PATCH 06/19] Move Upsert/RMW/Delete calls to PSF to be outside the UnsafeResume/SuspendThread --- cs/src/core/ClientSession/ClientSession.cs | 34 +++++++++++++++-- cs/src/core/Index/FASTER/FASTER.cs | 44 ++++++++-------------- cs/src/core/Index/FASTER/FASTERLegacy.cs | 9 +++-- cs/src/core/Index/PSF/PSFManager.cs | 3 -- 4 files changed, 53 insertions(+), 37 deletions(-) diff --git a/cs/src/core/ClientSession/ClientSession.cs b/cs/src/core/ClientSession/ClientSession.cs index ed546a007..fce9ec735 100644 --- a/cs/src/core/ClientSession/ClientSession.cs +++ b/cs/src/core/ClientSession/ClientSession.cs @@ -112,15 +112,29 @@ public Status Read(ref Key key, ref Input input, ref Output output, Context user [MethodImpl(MethodImplOptions.AggressiveInlining)] public Status Upsert(ref Key key, ref Value desiredValue, Context userContext, long serialNo) { + var updateArgs = new PSFUpdateArgs(); + FasterKVProviderData providerData = null; + Status status; + if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextUpsert(ref key, ref desiredValue, userContext, serialNo, ctx); + status = fht.ContextUpsert(ref key, ref desiredValue, userContext, serialNo, ctx, ref updateArgs); + if (status == Status.OK && this.fht.PSFManager.HasPSFs) + { + providerData = updateArgs.ChangeTracker is null + ? new FasterKVProviderData(this.fht.hlog, ref key, ref desiredValue) + : updateArgs.ChangeTracker.AfterData; + } } finally { if (SupportAsync) UnsafeSuspendThread(); } + + return providerData is null + ? status + : this.fht.PSFManager.Upsert(providerData, updateArgs.LogicalAddress, updateArgs.ChangeTracker); } /// @@ -167,15 +181,22 @@ static async ValueTask SlowUpsertAsync( [MethodImpl(MethodImplOptions.AggressiveInlining)] public Status RMW(ref Key key, ref Input input, Context userContext, long serialNo) { + var updateArgs = new PSFUpdateArgs(); + Status status; + if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextRMW(ref key, ref input, userContext, serialNo, ctx); + status = fht.ContextRMW(ref key, ref input, userContext, serialNo, ctx, ref updateArgs); } finally { if (SupportAsync) UnsafeSuspendThread(); } + + return status == Status.OK && this.fht.PSFManager.HasPSFs + ? this.fht.PSFManager.Update(updateArgs.ChangeTracker) + : status; } /// @@ -223,15 +244,22 @@ static async ValueTask SlowRMWAsync( [MethodImpl(MethodImplOptions.AggressiveInlining)] public Status Delete(ref Key key, Context userContext, long serialNo) { + var updateArgs = new PSFUpdateArgs(); + Status status; + if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextDelete(ref key, userContext, serialNo, ctx); + status = fht.ContextDelete(ref key, userContext, serialNo, ctx, ref updateArgs); } finally { if (SupportAsync) UnsafeSuspendThread(); } + + return status == Status.OK && this.fht.PSFManager.HasPSFs + ? this.fht.PSFManager.Delete(updateArgs.ChangeTracker) + : status; } /// diff --git a/cs/src/core/Index/FASTER/FASTER.cs b/cs/src/core/Index/FASTER/FASTER.cs index af8cd1104..d99f1e069 100644 --- a/cs/src/core/Index/FASTER/FASTER.cs +++ b/cs/src/core/Index/FASTER/FASTER.cs @@ -317,28 +317,21 @@ internal Status ContextRead(ref Key key, ref Input input, ref Output output, Con return status; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] internal Status ContextUpsert(ref Key key, ref Value value, Context context, long serialNo, - FasterExecutionContext sessionCtx) + FasterExecutionContext sessionCtx, ref PSFUpdateArgs psfUpdateArgs) { var pcontext = default(PendingContext); - var updateArgs = new PSFUpdateArgs(); var internalStatus = InternalUpsert(ref key, ref value, ref context, ref pcontext, sessionCtx, - serialNo, ref updateArgs); + serialNo, ref psfUpdateArgs); Status status; - if (internalStatus == OperationStatus.SUCCESS && this.PSFManager.HasPSFs) + if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { - status = this.PSFManager.Upsert(updateArgs.ChangeTracker is null - ? new FasterKVProviderData(this.hlog, ref key, ref value) - : updateArgs.ChangeTracker.AfterData, - updateArgs.LogicalAddress, updateArgs.ChangeTracker); + status = (Status)internalStatus; } else { - status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND - ? (Status)internalStatus - : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + status = HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); } sessionCtx.serialNum = serialNo; @@ -347,22 +340,19 @@ internal Status ContextUpsert(ref Key key, ref Value value, Context context, lon [MethodImpl(MethodImplOptions.AggressiveInlining)] internal Status ContextRMW(ref Key key, ref Input input, Context context, long serialNo, - FasterExecutionContext sessionCtx) + FasterExecutionContext sessionCtx, ref PSFUpdateArgs psfUpdateArgs) { var pcontext = default(PendingContext); - var updateArgs = new PSFUpdateArgs(); var internalStatus = InternalRMW(ref key, ref input, ref context, ref pcontext, sessionCtx, - serialNo, ref updateArgs); + serialNo, ref psfUpdateArgs); Status status; - if (internalStatus == OperationStatus.SUCCESS && this.PSFManager.HasPSFs) + if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { - status = this.PSFManager.Update(updateArgs.ChangeTracker); + status = (Status)internalStatus; } else { - status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND - ? (Status)internalStatus - : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + status = HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); } sessionCtx.serialNum = serialNo; @@ -370,23 +360,21 @@ internal Status ContextRMW(ref Key key, ref Input input, Context context, long s } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextDelete(ref Key key, Context context, long serialNo, FasterExecutionContext sessionCtx) + internal Status ContextDelete(ref Key key, Context context, long serialNo, FasterExecutionContext sessionCtx, + ref PSFUpdateArgs psfUpdateArgs) { var pcontext = default(PendingContext); - var updateArgs = new PSFUpdateArgs(); var internalStatus = InternalDelete(ref key, ref context, ref pcontext, sessionCtx, - serialNo, ref updateArgs); + serialNo, ref psfUpdateArgs); Status status; - if (internalStatus == OperationStatus.SUCCESS && this.PSFManager.HasPSFs) + if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { - status = this.PSFManager.Delete(updateArgs.ChangeTracker); + status = (Status)internalStatus; } else { - status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND - ? (Status)internalStatus - : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); + status = HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus); } sessionCtx.serialNum = serialNo; diff --git a/cs/src/core/Index/FASTER/FASTERLegacy.cs b/cs/src/core/Index/FASTER/FASTERLegacy.cs index fbf4a1b16..e87967b6f 100644 --- a/cs/src/core/Index/FASTER/FASTERLegacy.cs +++ b/cs/src/core/Index/FASTER/FASTERLegacy.cs @@ -114,7 +114,8 @@ public Status Read(ref Key key, ref Input input, ref Output output, Context cont [Obsolete("Use NewSession() and invoke Upsert() on the session.")] public Status Upsert(ref Key key, ref Value value, Context context, long serialNo) { - return ContextUpsert(ref key, ref value, context, serialNo, threadCtx.Value); + PSFUpdateArgs psfUpdateArgs = default; + return ContextUpsert(ref key, ref value, context, serialNo, threadCtx.Value, ref psfUpdateArgs); } /// @@ -128,7 +129,8 @@ public Status Upsert(ref Key key, ref Value value, Context context, long serialN [Obsolete("Use NewSession() and invoke RMW() on the session.")] public Status RMW(ref Key key, ref Input input, Context context, long serialNo) { - return ContextRMW(ref key, ref input, context, serialNo, threadCtx.Value); + PSFUpdateArgs psfUpdateArgs = default; + return ContextRMW(ref key, ref input, context, serialNo, threadCtx.Value, ref psfUpdateArgs); } /// @@ -144,7 +146,8 @@ public Status RMW(ref Key key, ref Input input, Context context, long serialNo) [Obsolete("Use NewSession() and invoke Delete() on the session.")] public Status Delete(ref Key key, Context context, long serialNo) { - return ContextDelete(ref key, context, serialNo, threadCtx.Value); + PSFUpdateArgs psfUpdateArgs = default; + return ContextDelete(ref key, context, serialNo, threadCtx.Value, ref psfUpdateArgs); } /// diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index 91009fef6..cb34688b4 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -24,9 +24,6 @@ internal Status Upsert(TProviderData data, TRecordId recordId, // TODO: RecordId locking, to ensure consistency of multiple PSFs if the same record is updated // multiple times; possibly a single Array[N] which is locked on TRecordId.GetHashCode % N. - // TODO: This is called from ContextUpsert or InternalCompleteRetryRequest, where it's still - // in the Context threading control for the primaryKV. I think it needs to move out of there. - // This Upsert was an Insert: For the FasterKV Insert fast path, changeTracker is null. if (changeTracker is null || changeTracker.UpdateOp == UpdateOperation.Insert) { From 381e83e775d8cfdf6025dff6689fc5b72503823a Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Mon, 8 Jun 2020 02:19:18 -0700 Subject: [PATCH 07/19] - RETRY_LATER if ReadAddress has to go to the disk during Phase.PREPARE - Remove Unsafe(Resume|Suspend)Thread in session.QueryPSF calls (it's done at a fine-grained level by session.PsfRead*) - Handle PENDING or RETRY of reads in QueryPSF - Add more TODOs --- cs/src/core/Index/FASTER/FASTERThread.cs | 9 +- cs/src/core/Index/PSF/FasterPSFImpl.cs | 28 ++--- .../Index/PSF/FasterPSFSessionOperations.cs | 111 ++++++------------ cs/src/core/Index/PSF/PSFChangeTracker.cs | 4 +- cs/src/core/Index/PSF/PSFFunctions.cs | 8 +- cs/src/core/Index/PSF/PSFGroup.cs | 25 ++-- cs/src/core/Index/PSF/PSFManager.cs | 10 +- cs/src/core/Index/PSF/PSFOutput.cs | 39 ++---- 8 files changed, 91 insertions(+), 143 deletions(-) diff --git a/cs/src/core/Index/FASTER/FASTERThread.cs b/cs/src/core/Index/FASTER/FASTERThread.cs index 8e98bd51b..e393ec49c 100644 --- a/cs/src/core/Index/FASTER/FASTERThread.cs +++ b/cs/src/core/Index/FASTER/FASTERThread.cs @@ -228,8 +228,15 @@ internal void InternalCompleteRetryRequest(FasterExecutionContext opCtx, FasterE internalStatus = InternalDelete(ref key, ref pendingContext.userContext, ref pendingContext, currentCtx, pendingContext.serialNum, ref pendingContext.psfUpdateArgs); break; + case OperationType.PSF_READ_ADDRESS: + // This is a special case for RECORD_ON_DISK in ContextPsfReadAddress during CPR; + // it is the only Read operation that goes through Retry. + internalStatus = this.PsfInternalReadAddress(ref pendingContext.psfReadArgs, ref pendingContext, + currentCtx, pendingContext.serialNum); + break; + case OperationType.PSF_READ_KEY: case OperationType.READ: - throw new FasterException("Cannot happen!"); + throw new FasterException("Reads go through the Pending route, not Retry, so this cannot happen!"); } var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index b2781ea88..9227b3b8f 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -169,7 +169,6 @@ internal OperationStatus PsfInternalReadAddress( var psfInput = psfArgs.Input; var psfOutput = psfArgs.Output; - // TODO: Verify we should always find the LogicalAddress OperationStatus status; #region Look up record in in-memory HybridLog @@ -221,29 +220,16 @@ ref hlog.GetValue(physicalAddress), // On-Disk Region else if (logicalAddress >= hlog.BeginAddress) { - status = OperationStatus.RECORD_ON_DISK; -#if false // TODO: See discussion in Teams/email; need to get the key here so we can get the hash, latch, etc. - if (sessionCtx.phase == Phase.PREPARE) - { - Debug.Assert(heldOperation != LatchOperation.Exclusive); - if (heldOperation == LatchOperation.Shared || HashBucket.TryAcquireSharedLatch(bucket)) - heldOperation = LatchOperation.Shared; - else - status = OperationStatus.CPR_SHIFT_DETECTED; - - if (RelaxedCPR) // don't hold on to shared latched during IO - { - if (heldOperation == LatchOperation.Shared) - HashBucket.ReleaseSharedLatch(bucket); - heldOperation = LatchOperation.None; - } - } -#endif + // We do not have a key here, so we cannot get the hash, latch, etc. for CPR and must retry later; + // this is the only Read operation that goes through Retry rather than pending. TODOtest: Retry + status = sessionCtx.phase == Phase.PREPARE + ? OperationStatus.RETRY_LATER + : OperationStatus.RECORD_ON_DISK; goto CreatePendingContext; } else { - // No record found + // No record found. TODOerr: we should always find the LogicalAddress return OperationStatus.NOTFOUND; } @@ -317,7 +303,7 @@ internal OperationStatus PsfInternalInsert( { logicalAddress = entry.Address; - // TODO: Test this: If this fails for any TPSFKey in the composite key, we'll create the pending + // TODOtest: If this fails for any TPSFKey in the composite key, we'll create the pending // context and come back here on the retry and overwrite any previously-obtained logicalAddress // at the chainLinkPtr. if (UseReadCache && ReadFromCache(ref compositeKey, ref logicalAddress, ref physicalAddress, diff --git a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs index b54316ba4..4ece8fa23 100644 --- a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs @@ -2,8 +2,8 @@ // Licensed under the MIT license. using System; +using System.Collections.Concurrent; using System.Collections.Generic; -using System.Linq; namespace FASTER.core { @@ -92,26 +92,37 @@ internal Status PsfDelete(ref Key key, ref Value value #region PSF Query API for primary FasterKV - internal FasterKVProviderData CreateProviderData(long logicalAddress) + internal Status CreateProviderData(long logicalAddress, + ConcurrentQueue> providerDatas) { // Looks up logicalAddress in the primary FasterKV - var psfArgs = new PSFReadArgs(new PSFInputPrimaryReadAddress(logicalAddress), - new PSFOutputPrimaryReadAddress(this.fht.hlog)); - - // Call this directly here because we are within UnsafeResumeThread. TODO: is that OK with an Iterator? - var status = fht.ContextPsfReadAddress(ref psfArgs, this.ctx.serialNum + 1, ctx); - var primaryOutput = psfArgs.Output as IPSFPrimaryOutput>; - return status == Status.OK ? primaryOutput.ProviderData : null; // TODO check other status + var primaryOutput = new PSFOutputPrimaryReadAddress(this.fht.hlog, providerDatas); + var psfArgs = new PSFReadArgs(new PSFInputPrimaryReadAddress(logicalAddress), primaryOutput); + return this.PsfReadAddress(ref psfArgs, this.ctx.serialNum + 1); } internal IEnumerable> ReturnProviderDatas(IEnumerable logicalAddresses) { + // If the record is on disk the Read will go pending and we will not receive it "synchronously" + // here; instead, it will work its way through the pending read system and call psfOutput.Visit. + // providerDatas gives that a place to put the record. We should encounter this only after all + // non-pending records have been read, but this approach allows any combination of pending and + // non-pending reads. + var providerDatas = new ConcurrentQueue>(); foreach (var logicalAddress in logicalAddresses) { - var providerData = this.CreateProviderData(logicalAddress); - if (!(providerData is null)) + var status = this.CreateProviderData(logicalAddress, providerDatas); + if (status == Status.ERROR) + { + // TODOerr: Handle error status from PsfReadAddress + } + while (providerDatas.TryDequeue(out var providerData)) yield return providerData; } + + this.CompletePending(spinWait: true); + while (providerDatas.TryDequeue(out var providerData)) + yield return providerData; } /// @@ -130,16 +141,8 @@ public IEnumerable> QueryPSF( PSF psf, TPSFKey psfKey, PSFQuerySettings querySettings = null) where TPSFKey : struct { - // Called on the secondary FasterKV - if (SupportAsync) UnsafeResumeThread(); - try - { - return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, psfKey, querySettings)); - } - finally - { - if (SupportAsync) UnsafeSuspendThread(); - } + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, psfKey, querySettings)); } /// @@ -159,16 +162,8 @@ public IEnumerable> QueryPSF (PSF psf, TPSFKey[] psfKeys, PSFQuerySettings querySettings = null) where TPSFKey : struct { - // Called on the secondary FasterKV - if (SupportAsync) UnsafeResumeThread(); - try - { - return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, psfKeys, querySettings)); - } - finally - { - if (SupportAsync) UnsafeSuspendThread(); - } + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, psfKeys, querySettings)); } /// @@ -198,17 +193,9 @@ public IEnumerable> QueryPSF @@ -241,17 +228,9 @@ public IEnumerable> QueryPSF @@ -280,16 +259,8 @@ public IEnumerable> QueryPSF( PSFQuerySettings querySettings = null) where TPSFKey : struct { - // Called on the secondary FasterKV - if (SupportAsync) UnsafeResumeThread(); - try - { - return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psfsAndKeys, matchPredicate, querySettings)); - } - finally - { - if (SupportAsync) UnsafeSuspendThread(); - } + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psfsAndKeys, matchPredicate, querySettings)); } /// @@ -327,17 +298,9 @@ public IEnumerable> QueryPSF _, ref PSFValue => src.CopyTo(ref dst, this.psfValueAccessor.RecordIdSize, this.psfValueAccessor.PSFCount); public void UpsertCompletionCallback(ref PSFCompositeKey _, ref PSFValue value, PSFContext ctx) - { /* TODO */ } + { /* TODO: UpsertCompletionCallback */ } #endregion Upserts #region Reads @@ -46,7 +46,7 @@ public unsafe void SingleReader(ref PSFCompositeKey _, ref PSFInputSeco => throw new PSFInternalErrorException("PSFOutput.Visit instead of SingleReader should be called on PSF-implementing FasterKVs"); public void ReadCompletionCallback(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFOutputSecondary output, PSFContext ctx, Status status) - { /* TODO */ } + { /* TODO: ReadCompletionCallback */ } #endregion Reads #region RMWs @@ -64,9 +64,9 @@ public void RMWCompletionCallback(ref PSFCompositeKey _, ref PSFInputSe #endregion RMWs public void DeleteCompletionCallback(ref PSFCompositeKey _, PSFContext ctx) - { /* TODO */ } + { /* TODO: DeleteCompletionCallback */ } public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) - { /* TODO */ } + { /* TODO: CheckpointCompletionCallback */ } } } diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index b37eab3d5..08aaf703c 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -187,6 +187,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor { // Note: stackalloc is safe because PendingContext or PSFChangeTracker will copy it to the bufferPool // if needed. On the Insert fast path, we don't want any allocations otherwise; changeTracker is null. + // TODO: Max psfCount per group to ensure stackalloc doesn't overflow var keyMemLen = ((keySize * this.PSFCount + sizeof(int) - 1) & ~(sizeof(int) - 1)); var keyMemInt = stackalloc int[keyMemLen / sizeof(int)]; for (var ii = 0; ii < keyMemLen; ++ii) @@ -243,7 +244,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor ref GroupKeysPair groupKeysPair = ref changeTracker.GetGroupRef(groupOrdinal); StoreKeys(ref groupKeysPair.After, keyMem, keyMemLen, flags, flagsMemLen); this.MarkChanges(groupKeysPair); - // TODO in debug, for initial dev, follow chains to assert the values match what is in the record's compositeKey + // TODOtest: In debug, for initial dev, follow chains to assert the values match what is in the record's compositeKey if (!groupKeysPair.HasChanges) return Status.OK; } @@ -303,7 +304,7 @@ public Status Update(PSFChangeTracker changeTracker) { if (this.GetBeforeKeys(changeTracker) != Status.OK) { - // TODO handle errors from GetBeforeKeys + // TODOerr: handle errors from GetBeforeKeys } } return this.ExecuteAndStore(changeTracker.AfterData, default, PSFExecutePhase.PostUpdate, changeTracker); @@ -344,20 +345,24 @@ public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key) private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKeyPtr, PSFInputSecondary input) { + // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them + // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr + var secondaryOutput = new PSFOutputSecondary(this.psfValueAccessor); var readArgs = new PSFReadArgs, PSFValue>( - input, new PSFOutputSecondary(this.psfValueAccessor)); + input, secondaryOutput); var session = this.GetSession(); - Status status; HashSet deadRecs = null; try { - status = session.PsfReadKey(ref queryKeyPtr.GetRef(), ref readArgs, session.ctx.serialNum + 1); - if (status != Status.OK) // TODO check other status + // Because we traverse the chain, we must wait for any pending read operations to complete. + // TODOperf: See if there is a better solution than spinWaiting in CompletePending. + Status status = session.PsfReadKey(ref queryKeyPtr.GetRef(), ref readArgs, session.ctx.serialNum + 1); + if (status == Status.PENDING) + session.CompletePending(spinWait: true); + if (status != Status.OK) // TODOerr: check other status yield break; - var secondaryOutput = readArgs.Output as IPSFSecondaryOutput; - if (secondaryOutput.Tombstone) { deadRecs ??= new HashSet(); @@ -372,7 +377,9 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe { readArgs.Input.ReadLogicalAddress = secondaryOutput.PreviousLogicalAddress; status = session.PsfReadAddress(ref readArgs, session.ctx.serialNum + 1); - if (status != Status.OK) // TODO check other status + if (status == Status.PENDING) + session.CompletePending(spinWait: true); + if (status != Status.OK) // TODOerr: check other status yield break; if (secondaryOutput.Tombstone) diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index cb34688b4..c0c37c6e3 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -33,7 +33,7 @@ internal Status Upsert(TProviderData data, TRecordId recordId, var status = group.ExecuteAndStore(data, recordId, PSFExecutePhase.Insert, changeTracker); if (status != Status.OK) { - // TODO handle errors + // TODOerr: handle errors } } return Status.OK; @@ -45,14 +45,13 @@ internal Status Upsert(TProviderData data, TRecordId recordId, internal Status Update(PSFChangeTracker changeTracker) { - // TODO: same comment as Insert re: Context threading control for the primaryKV changeTracker.PrepareGroups(this.psfGroups.Count); foreach (var group in this.psfGroups.Values) { var status = group.Update(changeTracker); if (status != Status.OK) { - // TODO handle errors + // TODOerr: handle errors } } return Status.OK; @@ -60,14 +59,13 @@ internal Status Update(PSFChangeTracker changeTracker) internal Status Delete(PSFChangeTracker changeTracker) { - // TODO: same comment as Insert re: Context threading control for the primaryKV changeTracker.PrepareGroups(this.psfGroups.Count); foreach (var group in this.psfGroups.Values) { var status = group.Delete(changeTracker); if (status != Status.OK) { - // TODO handle errors + // TODOerr: handle errors } } return Status.OK; @@ -187,7 +185,7 @@ internal IEnumerable QueryPSF(PSF psf, // are handled; the semantics are that a Union (via stream merge) of all records for all keys in the // array is done. Obviously there will be a tradeoff between the granularity of the bins and the // overhead of the PQ for the streams returned. - return QueryPSF(psf, psfKeys[0], querySettings); // TODO just to make the compiler happy + return QueryPSF(psf, psfKeys[0], querySettings); } internal IEnumerable QueryPSF( diff --git a/cs/src/core/Index/PSF/PSFOutput.cs b/cs/src/core/Index/PSF/PSFOutput.cs index 0d230ade2..98db30d04 100644 --- a/cs/src/core/Index/PSF/PSFOutput.cs +++ b/cs/src/core/Index/PSF/PSFOutput.cs @@ -3,6 +3,7 @@ #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member +using System.Collections.Concurrent; using System.Runtime.CompilerServices; namespace FASTER.core @@ -19,67 +20,52 @@ public interface IPSFOutput PSFOperationStatus Visit(int psfOrdinal, ref TKey key, ref TValue value, bool tombstone, bool isConcurrent); } - public interface IPSFPrimaryOutput - { - TProviderData ProviderData { get; } - } - /// /// Output from ReadInternal on the primary (stores user values) FasterKV instance when reading /// based on a LogicalAddress rather than a key. This class is FasterKV-provider-specific. /// /// The type of the key for user values /// The type of the user values - public unsafe class PSFOutputPrimaryReadAddress : IPSFOutput, - IPSFPrimaryOutput> + public unsafe class PSFOutputPrimaryReadAddress : IPSFOutput where TKVKey : new() where TKVValue : new() { private readonly AllocatorBase allocator; - public FasterKVProviderData ProviderData { get; private set; } + internal ConcurrentQueue> ProviderDatas { get; private set; } - internal PSFOutputPrimaryReadAddress(AllocatorBase alloc) + internal PSFOutputPrimaryReadAddress(AllocatorBase alloc, + ConcurrentQueue> provDatas) { this.allocator = alloc; - this.ProviderData = default; + this.ProviderDatas = provDatas; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public PSFOperationStatus Visit(int psfOrdinal, ref TKVKey key, ref TKVValue value, bool tombstone, bool isConcurrent) { // tombstone is not used here; it is only needed for the chains in the secondary FKV. - this.ProviderData = new FasterKVProviderData(this.allocator, ref key, ref value); + this.ProviderDatas.Enqueue(new FasterKVProviderData(this.allocator, ref key, ref value)); return new PSFOperationStatus(OperationStatus.SUCCESS); } } - public interface IPSFSecondaryOutput - { - TRecordId RecordId { get; } - - bool Tombstone { get; } - - long PreviousLogicalAddress { get; } - } - /// /// Output from operations on the secondary FasterKV instance (stores PSF chains). /// /// The type of the key returned from a /// The type of the provider's record identifier - public unsafe class PSFOutputSecondary : IPSFOutput, PSFValue>, - IPSFSecondaryOutput + public unsafe class PSFOutputSecondary : IPSFOutput, PSFValue> where TPSFKey : struct where TRecordId : struct { - IPSFValueAccessor> psfValueAccessor; + readonly IPSFValueAccessor> psfValueAccessor; - public TRecordId RecordId { get; private set; } + internal TRecordId RecordId { get; private set; } - public bool Tombstone { get; private set; } + internal bool Tombstone { get; private set; } - public long PreviousLogicalAddress { get; private set; } + internal long PreviousLogicalAddress { get; private set; } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal PSFOutputSecondary(IPSFValueAccessor> accessor) @@ -93,6 +79,7 @@ internal PSFOutputSecondary(IPSFValueAccessor> accessor) public PSFOperationStatus Visit(int psfOrdinal, ref PSFCompositeKey key, ref PSFValue value, bool tombstone, bool isConcurrent) { + // This is the secondary FKV so we wait to create provider data until QueryPSF returns. this.RecordId = value.RecordId; this.Tombstone = tombstone; this.PreviousLogicalAddress = *(this.psfValueAccessor.GetChainLinkPtrs(ref value) + psfOrdinal); From 0084058cdd687d8eb8baea6b09c5248e4f2884f5 Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Sun, 14 Jun 2020 01:10:02 -0700 Subject: [PATCH 08/19] - Fix multiple PSFs in a single group - Add binned category PSF to sample - Fix nullable return for Func<> PSF registrations - Add PSF_TRACE --- .../FasterPSFSample/BlittableOrders.cs | 5 +- cs/playground/FasterPSFSample/ColorKey.cs | 1 + cs/playground/FasterPSFSample/CombinedKey.cs | 51 ++++ cs/playground/FasterPSFSample/Constants.cs | 8 + cs/playground/FasterPSFSample/CountBinKey.cs | 31 +++ cs/playground/FasterPSFSample/FPSF.cs | 39 ++- .../FasterPSFSample/FasterPSFSample.cs | 251 +++++++++++++----- cs/playground/FasterPSFSample/IOrders.cs | 4 +- cs/playground/FasterPSFSample/ObjectOrders.cs | 17 +- cs/playground/FasterPSFSample/OrdersBinKey.cs | 29 -- cs/playground/FasterPSFSample/ParseArgs.cs | 8 + .../Properties/launchSettings.json | 8 + cs/playground/FasterPSFSample/SizeKey.cs | 1 + cs/src/core/Index/FASTER/FASTERImpl.cs | 38 ++- cs/src/core/Index/Interfaces/IFasterKV.cs | 10 +- .../core/Index/PSF/FasterKVPSFDefinition.cs | 2 +- cs/src/core/Index/PSF/FasterPSFImpl.cs | 65 +++-- .../core/Index/PSF/FasterPSFRegistration.cs | 6 +- cs/src/core/Index/PSF/PSFGroup.cs | 39 +-- cs/src/core/Index/PSF/PSFInput.cs | 8 +- cs/src/core/Index/PSF/PSFManager.cs | 2 + cs/src/core/Index/PSF/PSFValue.cs | 1 + 22 files changed, 453 insertions(+), 171 deletions(-) create mode 100644 cs/playground/FasterPSFSample/CombinedKey.cs create mode 100644 cs/playground/FasterPSFSample/CountBinKey.cs delete mode 100644 cs/playground/FasterPSFSample/OrdersBinKey.cs create mode 100644 cs/playground/FasterPSFSample/Properties/launchSettings.json diff --git a/cs/playground/FasterPSFSample/BlittableOrders.cs b/cs/playground/FasterPSFSample/BlittableOrders.cs index b08e1d056..bfc6c4fd5 100644 --- a/cs/playground/FasterPSFSample/BlittableOrders.cs +++ b/cs/playground/FasterPSFSample/BlittableOrders.cs @@ -3,7 +3,6 @@ using FASTER.core; using System; -using System.Drawing; namespace FasterPSFSample { @@ -16,9 +15,9 @@ public struct BlittableOrders : IOrders public int ColorArgb { get; set; } - public int NumSold { get; set; } + public int Count { get; set; } - public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {NumSold}"; + public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {Count}"; public class Functions : IFunctions, Context> { diff --git a/cs/playground/FasterPSFSample/ColorKey.cs b/cs/playground/FasterPSFSample/ColorKey.cs index 255adc3bb..7420fba8f 100644 --- a/cs/playground/FasterPSFSample/ColorKey.cs +++ b/cs/playground/FasterPSFSample/ColorKey.cs @@ -8,6 +8,7 @@ namespace FasterPSFSample { public struct ColorKey : IFasterEqualityComparer { + // Colors, strings, and enums are not blittable so we use int public int ColorArgb; public ColorKey(Color color) => this.ColorArgb = color.ToArgb(); diff --git a/cs/playground/FasterPSFSample/CombinedKey.cs b/cs/playground/FasterPSFSample/CombinedKey.cs new file mode 100644 index 000000000..b1a32133a --- /dev/null +++ b/cs/playground/FasterPSFSample/CombinedKey.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System.Drawing; + +namespace FasterPSFSample +{ + public struct CombinedKey : IFasterEqualityComparer + { + public int ValueType; + public int ValueInt; + + public CombinedKey(Constants.Size size) + { + this.ValueType = (int)Constants.ValueType.Size; + this.ValueInt = (int)size; + } + + public CombinedKey(Color color) + { + this.ValueType = (int)Constants.ValueType.Color; + this.ValueInt = color.ToArgb(); + } + + public CombinedKey(int countBin) + { + this.ValueType = (int)Constants.ValueType.Count; + this.ValueInt = countBin; + } + + public override string ToString() + { + var valueType = (Constants.ValueType)this.ValueType; + var valueTypeString = $"{nameof(Constants.ValueType)}.{valueType}"; + return valueType switch + { + Constants.ValueType.Size => $"{valueTypeString}: {(Constants.Size)this.ValueInt}", + Constants.ValueType.Color => $"{valueTypeString}: {Constants.ColorDict[this.ValueInt]}", + Constants.ValueType.Count => $"{valueTypeString}: {this.ValueInt}", + _ => throw new System.NotImplementedException("Unknown ValueType") + }; + } + + public long GetHashCode64(ref CombinedKey key) + => (long)key.ValueType << 32 | (uint)key.ValueInt.GetHashCode(); + + public bool Equals(ref CombinedKey k1, ref CombinedKey k2) + => k1.ValueType == k2.ValueType && k1.ValueInt == k2.ValueInt; + } +} diff --git a/cs/playground/FasterPSFSample/Constants.cs b/cs/playground/FasterPSFSample/Constants.cs index 80ad8d060..2483552e5 100644 --- a/cs/playground/FasterPSFSample/Constants.cs +++ b/cs/playground/FasterPSFSample/Constants.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. +using System; using System.Collections.Generic; using System.Drawing; @@ -29,5 +30,12 @@ public enum Size }; static internal Color[] Colors = { Color.Black, Color.Red, Color.Green, Color.Blue, Color.Purple }; + + public enum ValueType + { + Size, + Color, + Count + } } } diff --git a/cs/playground/FasterPSFSample/CountBinKey.cs b/cs/playground/FasterPSFSample/CountBinKey.cs new file mode 100644 index 000000000..2271b04e2 --- /dev/null +++ b/cs/playground/FasterPSFSample/CountBinKey.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; + +namespace FasterPSFSample +{ + public struct CountBinKey : IFasterEqualityComparer + { + internal const int BinSize = 100; + internal const int MaxOrders = BinSize * 10; + internal const int LastBin = 9; // 0-based + internal static bool WantLastBin; + + public int Bin; + + public CountBinKey(int bin) => this.Bin = bin; + + internal static bool GetBin(int numOrders, out int bin) + { + // Skip the last bin during initial inserts to illustrate not matching the PSF (returning null) + bin = numOrders / BinSize; + return WantLastBin || bin < LastBin; + } + + // Make the hashcode for this distinct from size enum values + public long GetHashCode64(ref CountBinKey key) => Utility.GetHashCode(key.Bin + 1000); + + public bool Equals(ref CountBinKey k1, ref CountBinKey k2) => k1.Bin == k2.Bin; + } +} diff --git a/cs/playground/FasterPSFSample/FPSF.cs b/cs/playground/FasterPSFSample/FPSF.cs index f963952be..7fa42f3ae 100644 --- a/cs/playground/FasterPSFSample/FPSF.cs +++ b/cs/playground/FasterPSFSample/FPSF.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using FASTER.core; +using System; #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member @@ -19,21 +20,45 @@ class FPSF internal PSF SizePsf; internal PSF ColorPsf; + internal PSF CountBinPsf; - internal FPSF(bool useObjectValues, bool useReadCache) + internal PSF CombinedSizePsf; + internal PSF CombinedColorPsf; + internal PSF CombinedCountBinPsf; + + internal FPSF(bool useObjectValues, bool useMultiGroup, bool useReadCache) { - this.logFiles = new LogFiles(useObjectValues, useReadCache, 2); + this.logFiles = new LogFiles(useObjectValues, useReadCache, useMultiGroup ? 3 : 1); this.fht = new FasterKV, TFunctions>( 1L << 20, new TFunctions(), this.logFiles.LogSettings, null, // TODO: add checkpoints useObjectValues ? new SerializerSettings { valueSerializer = () => new TSerializer() } : null); - var psfOrdinal = 0; - this.SizePsf = fht.RegisterPSF("sizePsf", (k, v) => new SizeKey((Constants.Size)v.SizeInt), - CreatePSFRegistrationSettings(psfOrdinal++)); - this.ColorPsf = fht.RegisterPSF("colorPsf", (k, v) => new ColorKey(Constants.ColorDict[v.ColorArgb]), - CreatePSFRegistrationSettings(psfOrdinal++)); + if (useMultiGroup) + { + var psfOrdinal = 0; + this.SizePsf = fht.RegisterPSF("sizePsf", (k, v) => new SizeKey((Constants.Size)v.SizeInt), + CreatePSFRegistrationSettings(psfOrdinal++)); + this.ColorPsf = fht.RegisterPSF("colorPsf", (k, v) => new ColorKey(Constants.ColorDict[v.ColorArgb]), + CreatePSFRegistrationSettings(psfOrdinal++)); + this.CountBinPsf = fht.RegisterPSF("countBinPsf", (k, v) => CountBinKey.GetBin(v.Count, out int bin) + ? new CountBinKey(bin) : (CountBinKey?)null, + CreatePSFRegistrationSettings(psfOrdinal++)); + } + else + { + var psfs = fht.RegisterPSF(new(string, Func)[] { + ("sizePsf", (k, v) => new CombinedKey((Constants.Size)v.SizeInt)), + ("colorPsf", (k, v) => new CombinedKey(Constants.ColorDict[v.ColorArgb])), + ("countBinPsf", (k, v) => CountBinKey.GetBin(v.Count, out int bin) + ? new CombinedKey(bin) : (CombinedKey?)null) + }, CreatePSFRegistrationSettings(0) + ); + this.CombinedSizePsf = psfs[0]; + this.CombinedColorPsf = psfs[1]; + this.CombinedCountBinPsf = psfs[2]; + } } PSFRegistrationSettings CreatePSFRegistrationSettings(int psfOrdinal) diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs index b6b249c7f..4930501c7 100644 --- a/cs/playground/FasterPSFSample/FasterPSFSample.cs +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. +//#define PSF_TRACE + using FASTER.core; using System; using System.Collections.Generic; @@ -14,16 +16,19 @@ namespace FasterPSFSample public partial class FasterPSFSample { private const int UpsertCount = 100; - private static int blueCount; - private static int mediumCount; - private static int mediumBlueCount; - + private static int blueCount, mediumCount, mediumBlueCount; + private static int bin7Count; + internal static Dictionary keyDict = new Dictionary(); + internal static List lastBinKeys = new List(); + private static int nextId = 1000000000; internal static int IPUColor; + internal static int serialNo; + static void Main(string[] argv) { if (!ParseArgs(argv)) @@ -42,14 +47,17 @@ internal static void RunSample() where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { - var fpsf = new FPSF(useObjectValue, useReadCache: true); + var fpsf = new FPSF(useObjectValue, useMultiGroups, useReadCache: true); try { + CountBinKey.WantLastBin = false; RunUpserts(fpsf); + CountBinKey.WantLastBin = true; RunReads(fpsf); var ok = QueryPSFs(fpsf) - && UpdateByUpsert(fpsf) - && UpdateByRMW(fpsf) + && UpdateSizeByUpsert(fpsf) + && UpdateColorByRMW(fpsf) + && UpdateCountByUpsert(fpsf) && Delete(fpsf); Console.WriteLine("--------------------------------------------------------"); Console.WriteLine(ok ? "Passed! All operations succeeded" @@ -61,10 +69,14 @@ internal static void RunSample() fpsf.Close(); } - Console.WriteLine("Press to end"); - Console.ReadLine(); + //Console.WriteLine("Press to end"); + //Console.ReadLine(); } + [Conditional("PSF_TRACE")] static void PsfTrace(string message) => Console.Write(message); + + [Conditional("PSF_TRACE")] static void PsfTraceLine(string message) => Console.WriteLine(message); + internal static void RunUpserts(FPSF fpsf) where TValue : IOrders, new() where TOutput : IOutput, new() @@ -79,15 +91,16 @@ internal static void RunUpserts(FPSF(FPSF(FPSF(FPSF fpsf) where TValue : IOrders, new() @@ -146,55 +169,81 @@ internal static bool QueryPSFs(FPSF[] providerDatas = null; var ok = true; - Console.WriteLine(); - Console.WriteLine("No join op; all Blues: "); - foreach (var providerData in session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Blue))) - { - ++actualCount; - ref TValue value = ref providerData.GetValue(); - if (verbose) - Console.WriteLine(indent + value); - } - ok &= blueCount == actualCount; - Console.WriteLine(blueCount == actualCount - ? $"Blue Passed: expected == actual ({blueCount})" - : $"Blue Failed: expected ({blueCount}) != actual ({actualCount})"); - actualCount = 0; - Console.WriteLine(); - Console.WriteLine("No join op; all Mediums: "); - foreach (var providerData in session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium))) + void verifyProviderDatas(string name, int expectedCount) { - ++actualCount; - ref TValue value = ref providerData.GetValue(); + Console.Write($"{indent4}All {name}: "); if (verbose) - Console.WriteLine(indent + value); + { + foreach (var providerData in providerDatas) + { + ref TValue value = ref providerData.GetValue(); + Console.WriteLine(indent4 + value); + } + } + ok &= expectedCount == providerDatas.Length; + Console.WriteLine(providerDatas.Length == expectedCount + ? $"Passed: expected == actual ({expectedCount})" + : $"Failed: expected ({expectedCount}) != actual ({providerDatas.Length})"); + } - ok &= mediumCount == actualCount; - Console.WriteLine(mediumCount == actualCount - ? $"Medium Passed: expected == actual ({mediumCount})" - : $"Medium Failed: expected ({mediumCount}) != actual ({actualCount})"); + + // TODO: Intersect/Union (mediumBlueDatas, etc.) + + providerDatas = useMultiGroups + ? session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium)).ToArray() + : session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium)).ToArray(); + verifyProviderDatas("Medium", mediumCount); + + providerDatas = useMultiGroups + ? session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Blue)).ToArray() + : session.QueryPSF(fpsf.CombinedColorPsf, new CombinedKey(Color.Blue)).ToArray(); + verifyProviderDatas("Blue", blueCount); + + providerDatas = useMultiGroups + ? session.QueryPSF(fpsf.CountBinPsf, new CountBinKey(7)).ToArray() + : session.QueryPSF(fpsf.CombinedCountBinPsf, new CombinedKey(7)).ToArray(); + verifyProviderDatas("Bin7", bin7Count); + + providerDatas = useMultiGroups + ? session.QueryPSF(fpsf.CountBinPsf, new CountBinKey(CountBinKey.LastBin)).ToArray() + : session.QueryPSF(fpsf.CombinedCountBinPsf, new CombinedKey(CountBinKey.LastBin)).ToArray(); + verifyProviderDatas("LastBin", 0); // Insert skipped (returned null from the PSF) all that fall into the last bin return ok; } - internal static bool UpdateByUpsert(FPSF fpsf) + private static void WriteResult(bool isInitial, string name, int expectedCount, int actualCount) + { + var tag = isInitial ? "Initial" : "Updated"; + Console.WriteLine(expectedCount == actualCount + ? $"{indent4}{tag} {name} Passed: expected == actual ({expectedCount})" + : $"{indent4}{tag} {name} Failed: expected ({expectedCount}) != actual ({actualCount})"); + } + + internal static bool UpdateSizeByUpsert(FPSF fpsf) where TValue : IOrders, new() where TOutput : IOutput, new() where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { Console.WriteLine(); - Console.WriteLine("Updating Sizes"); + Console.WriteLine("Updating Sizes via Upsert"); using var session = fpsf.fht.NewSession(); - var xxlDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.XXLarge)).ToArray(); - var mediumDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium)).ToArray(); + + FasterKVProviderData[] GetSizeDatas(Constants.Size size) + => useMultiGroups + ? session.QueryPSF(fpsf.SizePsf, new SizeKey(size)).ToArray() + : session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(size)).ToArray(); + + var xxlDatas = GetSizeDatas(Constants.Size.XXLarge); + WriteResult(isInitial: true, "XXLarge", 0, xxlDatas.Length); + var mediumDatas = GetSizeDatas(Constants.Size.Medium); + WriteResult(isInitial: true, "Medium", mediumCount, mediumDatas.Length); var expected = mediumDatas.Length; - Console.WriteLine(); - Console.Write($"Changing all Medium to XXLarge; initial counts Medium {mediumDatas.Length}, XXLarge {xxlDatas.Length}"); + Console.WriteLine($"{indent2}Changing all Medium to XXLarge"); var context = new Context(); foreach (var providerData in mediumDatas) @@ -205,33 +254,40 @@ internal static bool UpdateByUpsert(FP value.SizeInt = (int)Constants.Size.XXLarge; // Reuse the same key - session.Upsert(ref providerData.GetKey(), ref value, context, 2); + session.Upsert(ref providerData.GetKey(), ref value, context, serialNo); } - Console.WriteLine(); + ++serialNo; - xxlDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.XXLarge)).ToArray(); - mediumDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium)).ToArray(); + xxlDatas = GetSizeDatas(Constants.Size.XXLarge); + mediumDatas = GetSizeDatas(Constants.Size.Medium); bool ok = xxlDatas.Length == expected && mediumDatas.Length == 0; - Console.Write(ok ? "Passed" : "*** Failed *** "); - Console.WriteLine($": Medium {mediumDatas.Length}, XXLarge {xxlDatas.Length}"); + WriteResult(isInitial: false, "XXLarge", expected, xxlDatas.Length); + WriteResult(isInitial: false, "Medium", 0, mediumDatas.Length); return ok; } - internal static bool UpdateByRMW(FPSF fpsf) + internal static bool UpdateColorByRMW(FPSF fpsf) where TValue : IOrders, new() where TOutput : IOutput, new() where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { Console.WriteLine(); - Console.WriteLine("Updating Colors"); + Console.WriteLine("Updating Colors via RMW"); using var session = fpsf.fht.NewSession(); - var purpleDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Purple)).ToArray(); - var blueDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Blue)).ToArray(); + + FasterKVProviderData[] GetColorDatas(Color color) + => useMultiGroups + ? session.QueryPSF(fpsf.ColorPsf, new ColorKey(color)).ToArray() + : session.QueryPSF(fpsf.CombinedColorPsf, new CombinedKey(color)).ToArray(); + + var purpleDatas = GetColorDatas(Color.Purple); + WriteResult(isInitial: true, "Purple", 0, purpleDatas.Length); + var blueDatas = GetColorDatas(Color.Blue); + WriteResult(isInitial: true, "Blue", blueCount, blueDatas.Length); var expected = blueDatas.Length; - Console.WriteLine(); - Console.Write($"Changing all Blue to Purple; initial counts Blue {blueDatas.Length}, Purple {purpleDatas.Length}"); + Console.WriteLine($"{indent2}Changing all Blue to Purple"); IPUColor = Color.Purple.ToArgb(); var context = new Context(); @@ -239,15 +295,67 @@ internal static bool UpdateByRMW(FPSF< foreach (var providerData in blueDatas) { // This will call Functions<>.InPlaceUpdater. - session.RMW(ref providerData.GetKey(), ref input, context, 3); + session.RMW(ref providerData.GetKey(), ref input, context, serialNo); } - Console.WriteLine(); + ++serialNo; - purpleDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Purple)).ToArray(); - blueDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Blue)).ToArray(); + purpleDatas = GetColorDatas(Color.Purple); + blueDatas = GetColorDatas(Color.Blue); bool ok = purpleDatas.Length == expected && blueDatas.Length == 0; - Console.Write(ok ? "Passed" : "*** Failed *** "); - Console.WriteLine($": Blue {blueDatas.Length}, Purple {purpleDatas.Length}"); + WriteResult(isInitial: false, "Purple", expected, purpleDatas.Length); + WriteResult(isInitial: false, "Blue", 0, blueDatas.Length); + return ok; + } + + internal static bool UpdateCountByUpsert(FPSF fpsf) + where TValue : IOrders, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Updating Counts via Upsert"); + + using var session = fpsf.fht.NewSession(); + + var bin7 = 7; + FasterKVProviderData[] GetCountDatas(int bin) + => useMultiGroups + ? session.QueryPSF(fpsf.CountBinPsf, new CountBinKey(bin)).ToArray() + : session.QueryPSF(fpsf.CombinedCountBinPsf, new CombinedKey(bin)).ToArray(); + + // First show we've nothing in the last bin, and get all in bin7. + var lastBinDatas = GetCountDatas(CountBinKey.LastBin); + var ok = lastBinDatas.Length == 0; + WriteResult(isInitial: true, "LastBin", 0, lastBinDatas.Length); + + var bin7Datas = GetCountDatas(bin7); + ok &= lastBinDatas.Length == 0; + WriteResult(isInitial: true, "Bin7", bin7Count, bin7Datas.Length); + + Console.WriteLine($"{indent2}Changing all Bin7 to LastBin"); + var context = new Context(); + foreach (var providerData in bin7Datas) + { + // Update the value + ref TValue value = ref providerData.GetValue(); + Debug.Assert(CountBinKey.GetBin(value.Count, out int tempBin) && tempBin == bin7); + value.Count += (CountBinKey.LastBin - bin7) * CountBinKey.BinSize; + Debug.Assert(CountBinKey.GetBin(value.Count, out tempBin) && tempBin == CountBinKey.LastBin); + + // Reuse the same key + session.Upsert(ref providerData.GetKey(), ref value, context, serialNo); + } + ++serialNo; + + var expectedLastBinCount = bin7Datas.Length; + lastBinDatas = GetCountDatas(CountBinKey.LastBin); + ok &= lastBinDatas.Length == expectedLastBinCount; + WriteResult(isInitial: false, "LastBin", expectedLastBinCount, lastBinDatas.Length); + + bin7Datas = GetCountDatas(bin7); + ok &= bin7Datas.Length == 0; + WriteResult(isInitial: false, "Bin7", 0, bin7Datas.Length); return ok; } @@ -261,7 +369,13 @@ internal static bool Delete(FPSF[] GetColorDatas(Color color) + => useMultiGroups + ? session.QueryPSF(fpsf.ColorPsf, new ColorKey(color)).ToArray() + : session.QueryPSF(fpsf.CombinedColorPsf, new CombinedKey(color)).ToArray(); + + var redDatas = GetColorDatas(Color.Red); Console.WriteLine(); Console.Write($"Deleting all Reds; initial count {redDatas.Length}"); @@ -269,11 +383,12 @@ internal static bool Delete(FPSF.InPlaceUpdater. - session.Delete(ref providerData.GetKey(), context, 4); + session.Delete(ref providerData.GetKey(), context, serialNo); } + ++serialNo; Console.WriteLine(); - redDatas = session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Red)).ToArray(); + redDatas = GetColorDatas(Color.Red); var ok = redDatas.Length == 0; Console.Write(ok ? "Passed" : "*** Failed *** "); Console.WriteLine($": Red {redDatas.Length}"); diff --git a/cs/playground/FasterPSFSample/IOrders.cs b/cs/playground/FasterPSFSample/IOrders.cs index 9da962336..0564ffe15 100644 --- a/cs/playground/FasterPSFSample/IOrders.cs +++ b/cs/playground/FasterPSFSample/IOrders.cs @@ -12,8 +12,8 @@ public interface IOrders int ColorArgb { get; set; } - int NumSold { get; set; } + int Count { get; set; } - (int, int, int, int) MemberTuple => (this.Id, this.SizeInt, this.ColorArgb, this.NumSold); + (int, int, int, int) MemberTuple => (this.Id, this.SizeInt, this.ColorArgb, this.Count); } } diff --git a/cs/playground/FasterPSFSample/ObjectOrders.cs b/cs/playground/FasterPSFSample/ObjectOrders.cs index 17011cae4..c0b87301e 100644 --- a/cs/playground/FasterPSFSample/ObjectOrders.cs +++ b/cs/playground/FasterPSFSample/ObjectOrders.cs @@ -8,26 +8,31 @@ namespace FasterPSFSample { public class ObjectOrders : IOrders { + const int numValues = 3; + const int sizeOrd = 0; + const int colorOrd = 1; + const int countOrd = 2; + public int Id { get; set; } // Colors, strings, and enums are not blittable so we use int - public int SizeInt { get => values[0]; set => values[0] = value; } + public int SizeInt { get => values[sizeOrd]; set => values[sizeOrd] = value; } - public int ColorArgb { get => values[1]; set => values[1] = value; } + public int ColorArgb { get => values[colorOrd]; set => values[colorOrd] = value; } - public int NumSold { get => values[2]; set => values[2] = value; } + public int Count { get => values[countOrd]; set => values[countOrd] = value; } - public int[] values; + public int[] values = new int[numValues]; public ObjectOrders() => throw new InvalidOperationException("Must use ctor overload"); - public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {NumSold}"; + public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {Count}"; public class Serializer : BinaryObjectSerializer { public override void Deserialize(ref ObjectOrders obj) { - obj.values = new int[3]; + obj.values = new int[numValues]; for (var ii = 0; ii < obj.values.Length; ++ii) obj.values[ii] = reader.ReadInt32(); } diff --git a/cs/playground/FasterPSFSample/OrdersBinKey.cs b/cs/playground/FasterPSFSample/OrdersBinKey.cs deleted file mode 100644 index 87db1b150..000000000 --- a/cs/playground/FasterPSFSample/OrdersBinKey.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -using FASTER.core; - -namespace FasterPSFSample -{ - public struct OrdersBinKey : IFasterEqualityComparer - { - internal const int BinSize = 100; - internal const int MaxOrders = BinSize * 10; - internal const int MaxBin = 8; // 0-based; skip the last one - - public int Bin; - - public OrdersBinKey(int bin) => this.Bin = bin; - - internal bool GetBin(int numOrders, out int bin) - { - bin = numOrders / BinSize; - return bin < MaxBin; - } - - // Make the hashcode for this distinct from size enum values - public long GetHashCode64(ref OrdersBinKey key) => Utility.GetHashCode(this.Bin + 1000); - - public bool Equals(ref OrdersBinKey k1, ref OrdersBinKey k2) => k1.Bin == k2.Bin; - } -} diff --git a/cs/playground/FasterPSFSample/ParseArgs.cs b/cs/playground/FasterPSFSample/ParseArgs.cs index 36e015959..dfa57943f 100644 --- a/cs/playground/FasterPSFSample/ParseArgs.cs +++ b/cs/playground/FasterPSFSample/ParseArgs.cs @@ -8,9 +8,11 @@ namespace FasterPSFSample public partial class FasterPSFSample { private static bool useObjectValue; + private static bool useMultiGroups; private static bool verbose; const string ObjValuesArg = "--objValues"; + const string MultiGroupArg = "--multiGroup"; static bool ParseArgs(string[] argv) { @@ -20,6 +22,7 @@ static bool Usage(string message = null) Console.WriteLine($"Usage: Run one or more Predicate Subset Functions (PSFs), specifying whether to use object or blittable (primitive) values."); Console.WriteLine(); Console.WriteLine($" {ObjValuesArg}: Use objects instead of blittable Value; default is {useObjectValue}"); + Console.WriteLine($" {MultiGroupArg}: Put each PSF in a separate group; default is {useMultiGroups}"); Console.WriteLine(); if (!string.IsNullOrEmpty(message)) { @@ -39,6 +42,11 @@ static bool Usage(string message = null) useObjectValue = true; continue; } + if (string.Compare(arg, MultiGroupArg, ignoreCase: true) == 0) + { + useMultiGroups = true; + continue; + } if (string.Compare(arg, "--help", ignoreCase: true) == 0 || arg == "/?" || arg == "-?") return Usage(); if (string.Compare(arg, "-v", ignoreCase: true) == 0) diff --git a/cs/playground/FasterPSFSample/Properties/launchSettings.json b/cs/playground/FasterPSFSample/Properties/launchSettings.json new file mode 100644 index 000000000..560f42813 --- /dev/null +++ b/cs/playground/FasterPSFSample/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "FasterPSFSample": { + "commandName": "Project", + "commandLineArgs": "--multiGroup" + } + } +} \ No newline at end of file diff --git a/cs/playground/FasterPSFSample/SizeKey.cs b/cs/playground/FasterPSFSample/SizeKey.cs index 1962248f9..ffd2a03fd 100644 --- a/cs/playground/FasterPSFSample/SizeKey.cs +++ b/cs/playground/FasterPSFSample/SizeKey.cs @@ -7,6 +7,7 @@ namespace FasterPSFSample { public struct SizeKey : IFasterEqualityComparer { + // Colors, strings, and enums are not blittable so we use int public int SizeInt; public SizeKey(Constants.Size size) => this.SizeInt = (int)size; diff --git a/cs/src/core/Index/FASTER/FASTERImpl.cs b/cs/src/core/Index/FASTER/FASTERImpl.cs index 946c9cf0f..afb2bff00 100644 --- a/cs/src/core/Index/FASTER/FASTERImpl.cs +++ b/cs/src/core/Index/FASTER/FASTERImpl.cs @@ -1838,6 +1838,34 @@ private void BlockAllocateReadCache(int recordSize, out long logicalAddress, Fas } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TraceBackForKeyMatch( + ref Key key, + long fromLogicalAddress, + long minOffset, + out long foundLogicalAddress, + out long foundPhysicalAddress) + { + foundLogicalAddress = fromLogicalAddress; + while (foundLogicalAddress >= minOffset) + { + foundPhysicalAddress = hlog.GetPhysicalAddress(foundLogicalAddress); + if (comparer.Equals(ref key, ref hlog.GetKey(foundPhysicalAddress))) + { + return true; + } + else + { + foundLogicalAddress = hlog.GetInfo(foundPhysicalAddress).PreviousAddress; + //This makes testing REALLY slow + //Debug.WriteLine("Tracing back"); + continue; + } + } + foundPhysicalAddress = Constants.kInvalidAddress; + return false; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool TraceBackForKeyMatch( ref Key key, @@ -1845,18 +1873,18 @@ private bool TraceBackForKeyMatch( long minOffset, out long foundLogicalAddress, out long foundPhysicalAddress, - IPSFInput psfInput = null) + IPSFInput psfInput) { foundLogicalAddress = fromLogicalAddress; while (foundLogicalAddress >= minOffset) { foundPhysicalAddress = hlog.GetPhysicalAddress(foundLogicalAddress); - if (psfInput is null - ? comparer.Equals(ref key, ref hlog.GetKey(foundPhysicalAddress)) - : psfInput.EqualsAt(ref key, ref hlog.GetKey(foundPhysicalAddress))) + if (psfInput.EqualsAt(ref key, ref hlog.GetKey(foundPhysicalAddress))) return true; - foundLogicalAddress = hlog.GetInfo(foundPhysicalAddress).PreviousAddress; + foundLogicalAddress = *(this.psfValueAccessor.GetChainLinkPtrs(ref hlog.GetValue(foundPhysicalAddress)) + + psfInput.PsfOrdinal); + //This makes testing REALLY slow //Debug.WriteLine("Tracing back"); continue; diff --git a/cs/src/core/Index/Interfaces/IFasterKV.cs b/cs/src/core/Index/Interfaces/IFasterKV.cs index a982ba129..b0ba24d4b 100644 --- a/cs/src/core/Index/Interfaces/IFasterKV.cs +++ b/cs/src/core/Index/Interfaces/IFasterKV.cs @@ -177,7 +177,7 @@ PSF[] RegisterPSF /// Optional registration settings for the secondary FasterKV instances, etc. /// A FasterKV-specific implementation whose TRecordId is long( PSF RegisterPSF( - string psfName, Func psfFunc, + string psfName, Func psfFunc, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct; @@ -194,7 +194,7 @@ PSF RegisterPSF( /// "params" won't allow the optional fromAddress and keyComparer, so an overload is provided /// to specify those PSF[] RegisterPSF( - params (string, Func)[] psfFuncs) + params (string, Func)[] psfFuncs) where TPSFKey : struct; /// @@ -202,10 +202,10 @@ PSF[] RegisterPSF( /// /// /// // Unfortunately the array type cannot be implicitly deduced in current versions of the compiler - /// var sizePsf = fht.RegisterPSF(new (string, Func{TKVKey, TKVValue, TPSFKey>)[] { + /// var sizePsf = fht.RegisterPSF(new (string, Func{TKVKey, TKVValue, TPSFKey})[] { /// ("sizePsf", (k, v) => new TPSFKey(v.size)), /// ("colorPsf", (k, v) => new TPSFKey(v.color))}, - /// keyComparer, fromAddress); + /// registrationSettings); /// /// The type of the key value returned from the /// One or more tuples containing a PSF name and implementing Func; the name must be @@ -214,7 +214,7 @@ PSF[] RegisterPSF( /// If the registrationSettings parameters are null, then it is simpler to call the "params" overload /// than to create the vector explicitly PSF[] RegisterPSF( - (string, Func)[] psfDefs, + (string, Func)[] psfDefs, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct; diff --git a/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs b/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs index 6b2ce1b4e..20d711e34 100644 --- a/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs +++ b/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs @@ -62,7 +62,7 @@ public FasterKVPSFDefinition(string name, PredicateFunc predicate) /// /// /// - public FasterKVPSFDefinition(string name, Func predicate) + public FasterKVPSFDefinition(string name, Func predicate) { TPSFKey? wrappedPredicate(ref TKVKey key, ref TKVValue value) => predicate(key, value); diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index 9227b3b8f..5343e69a8 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -1,6 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. +//#define PSF_TRACE + +using System; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Threading; @@ -68,8 +71,9 @@ ref readcache.GetValue(physicalAddress), if (!psfInput.EqualsAt(ref queryKey, ref hlog.GetKey(physicalAddress))) { - logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; - TraceBackForKeyMatch(ref queryKey, + logicalAddress = *(this.psfValueAccessor.GetChainLinkPtrs(ref hlog.GetValue(physicalAddress)) + psfInput.PsfOrdinal); + if (logicalAddress > hlog.BeginAddress) + TraceBackForKeyMatch(ref queryKey, logicalAddress, hlog.HeadAddress, out logicalAddress, @@ -255,6 +259,10 @@ ref hlog.GetValue(physicalAddress), return status; } + [Conditional("PSF_TRACE")] static void PsfTrace(string message) => Console.Write(message); + + [Conditional("PSF_TRACE")] static void PsfTraceLine(string message) => Console.WriteLine(message); + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal OperationStatus PsfInternalInsert( ref Key compositeKey, ref Value value, ref Input input, @@ -278,19 +286,22 @@ internal OperationStatus PsfInternalInsert( var psfCount = this.psfValueAccessor.PSFCount; long* hashes = stackalloc long[psfCount]; long* chainLinkPtrs = this.psfValueAccessor.GetChainLinkPtrs(ref value); - for (var chainLinkIdx = 0; chainLinkIdx < psfCount; ++chainLinkIdx) + PsfTrace($" {value} |"); + for (psfInput.PsfOrdinal = 0; psfInput.PsfOrdinal < psfCount; ++psfInput.PsfOrdinal) { // For RCU, or in case we had to retry due to CPR_SHIFT and somehow managed to delete // the previously found record, clear out the chain link pointer. - long* chainLinkPtr = chainLinkPtrs + chainLinkIdx; + long* chainLinkPtr = chainLinkPtrs + psfInput.PsfOrdinal; *chainLinkPtr = Constants.kInvalidAddress; - psfInput.PsfOrdinal = chainLinkIdx; if (psfInput.IsNullAt) + { + PsfTrace($" -0-"); continue; + } var hash = psfInput.GetHashCode64At(ref compositeKey); - *(hashes + chainLinkIdx) = hash; + *(hashes + psfInput.PsfOrdinal) = hash; var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); if (sessionCtx.phase != Phase.REST) @@ -298,10 +309,10 @@ internal OperationStatus PsfInternalInsert( #region Trace back for record in in-memory HybridLog entry = default; - var tagExists = FindTag(hash, tag, ref bucket, ref slot, ref entry); - if (tagExists) + if (FindTag(hash, tag, ref bucket, ref slot, ref entry)) { logicalAddress = entry.Address; + PsfTrace($" {logicalAddress}/"); // TODOtest: If this fails for any TPSFKey in the composite key, we'll create the pending // context and come back here on the retry and overwrite any previously-obtained logicalAddress @@ -324,31 +335,40 @@ internal OperationStatus PsfInternalInsert( if (!psfInput.EqualsAt(ref compositeKey, ref hlog.GetKey(physicalAddress))) { - logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; - TraceBackForKeyMatch(ref compositeKey, - logicalAddress, - hlog.HeadAddress, - out logicalAddress, - out physicalAddress, - psfInput); + logicalAddress = *(this.psfValueAccessor.GetChainLinkPtrs(ref hlog.GetValue(physicalAddress)) + psfInput.PsfOrdinal); + if (logicalAddress > hlog.BeginAddress) + TraceBackForKeyMatch(ref compositeKey, + logicalAddress, + hlog.HeadAddress, + out logicalAddress, + out physicalAddress, + psfInput); } } + PsfTrace($"{logicalAddress}"); + if (logicalAddress < hlog.BeginAddress) + continue; + if (hlog.GetInfo(physicalAddress).Tombstone) { // The chain might extend past a tombstoned record so we must include it in the chain - // unless its link at chainLinkIdx is kInvalidAddress. + // unless its prevLink at psfOrdinal is invalid. long* prevLinks = this.psfValueAccessor.GetChainLinkPtrs(ref hlog.GetValue(physicalAddress)); - long* prevLink = prevLinks + chainLinkIdx; - if (*prevLink == Constants.kInvalidAddress) + long* prevLink = prevLinks + psfInput.PsfOrdinal; + if (*prevLink < hlog.BeginAddress) continue; } *chainLinkPtr = logicalAddress; } + else + { + PsfTrace($" {logicalAddress}/{logicalAddress}"); + } #endregion } -#region Entry latch operation + #region Entry latch operation if (sessionCtx.phase != Phase.REST) { switch (sessionCtx.phase) @@ -435,13 +455,14 @@ internal OperationStatus PsfInternalInsert( hlog.ShallowCopy(ref compositeKey, ref hlog.GetKey(newPhysicalAddress)); functions.SingleWriter(ref compositeKey, ref value, ref hlog.GetValue(newPhysicalAddress)); - for (var chainLinkIdx = 0; chainLinkIdx < psfCount; ++chainLinkIdx) + PsfTraceLine($" | {newLogicalAddress}"); + for (var psfOrdinal = 0; psfOrdinal < psfCount; ++psfOrdinal) { - psfInput.PsfOrdinal = chainLinkIdx; + psfInput.PsfOrdinal = psfOrdinal; if (psfInput.IsNullAt) continue; - var hash = *(hashes + chainLinkIdx); + var hash = *(hashes + psfOrdinal); var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); entry = default; FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry, hlog.BeginAddress); diff --git a/cs/src/core/Index/PSF/FasterPSFRegistration.cs b/cs/src/core/Index/PSF/FasterPSFRegistration.cs index f5bb36905..3892f94b5 100644 --- a/cs/src/core/Index/PSF/FasterPSFRegistration.cs +++ b/cs/src/core/Index/PSF/FasterPSFRegistration.cs @@ -52,7 +52,7 @@ public PSF[] RegisterPSF /// public PSF RegisterPSF( - string psfName, Func psfFunc, + string psfName, Func psfFunc, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct => this.PSFManager.RegisterPSF(new FasterKVPSFDefinition(psfName, psfFunc), @@ -60,14 +60,14 @@ public PSF RegisterPSF( /// public PSF[] RegisterPSF( - params (string, Func)[] psfFuncs) + params (string, Func)[] psfFuncs) where TPSFKey : struct => this.PSFManager.RegisterPSF(psfFuncs.Select(e => new FasterKVPSFDefinition(e.Item1, e.Item2)).ToArray(), CreateDefaultRegistrationSettings()); /// public PSF[] RegisterPSF( - (string, Func)[] psfDefs, + (string, Func)[] psfDefs, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct => this.PSFManager.RegisterPSF(psfDefs.Select(e => new FasterKVPSFDefinition(e.Item1, e.Item2)).ToArray(), diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index 08aaf703c..07911cef2 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -163,20 +163,23 @@ private unsafe void StoreKeys(ref GroupKeys keys, byte* kPtr, int kLen, PSFResul keys.Set(poolKeyMem, flagsMem); } - internal unsafe void MarkChanges(GroupKeysPair keysPair) + internal unsafe void MarkChanges(ref GroupKeysPair keysPair) { - var before = keysPair.Before; - var after = keysPair.After; - var beforeCompKey = before.GetCompositeKeyRef>(); - var afterCompKey = after.GetCompositeKeyRef>(); + ref GroupKeys before = ref keysPair.Before; + ref GroupKeys after = ref keysPair.After; + ref PSFCompositeKey beforeCompKey = ref before.GetCompositeKeyRef>(); + ref PSFCompositeKey afterCompKey = ref after.GetCompositeKeyRef>(); for (var ii = 0; ii < this.PSFCount; ++ii) { - bool keysEqual() => beforeCompKey.GetKeyRef(ii, this.keySize).Equals(afterCompKey.GetKeyRef(ii, this.keySize)); + var beforeIsNull = before.IsNullAt(ii); + var afterIsNull = after.IsNullAt(ii); + var keysEqual = !beforeIsNull && !afterIsNull + && beforeCompKey.GetKeyRef(ii, this.keySize).Equals(afterCompKey.GetKeyRef(ii, this.keySize)); // IsNull is already set in PSFGroup.ExecuteAndStore. - if (!before.IsNullAt(ii) && (after.IsNullAt(ii) || !keysEqual())) + if (!before.IsNullAt(ii) && (after.IsNullAt(ii) || !keysEqual)) *after.ResultFlags |= PSFResultFlags.UnlinkOld; - if (!after.IsNullAt(ii) && (before.IsNullAt(ii) || !keysEqual())) + if (!after.IsNullAt(ii) && (before.IsNullAt(ii) || !keysEqual)) *after.ResultFlags |= PSFResultFlags.LinkNew; } } @@ -187,10 +190,10 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor { // Note: stackalloc is safe because PendingContext or PSFChangeTracker will copy it to the bufferPool // if needed. On the Insert fast path, we don't want any allocations otherwise; changeTracker is null. - // TODO: Max psfCount per group to ensure stackalloc doesn't overflow + // TODO: Cutoff (psfCount * keySize) per group to use bufferPool to ensure stackalloc doesn't overflow var keyMemLen = ((keySize * this.PSFCount + sizeof(int) - 1) & ~(sizeof(int) - 1)); var keyMemInt = stackalloc int[keyMemLen / sizeof(int)]; - for (var ii = 0; ii < keyMemLen; ++ii) + for (var ii = 0; ii < keyMemLen / sizeof(int); ++ii) keyMemInt[ii] = 0; var keyMem = (byte*)keyMemInt; ref PSFCompositeKey compositeKey = ref Unsafe.AsRef>(keyMem); @@ -201,7 +204,6 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor for (int ii = 0; ii < this.psfDefinitions.Length; ++ii) { var key = this.psfDefinitions[ii].Execute(providerData); - *(flags + ii) = key.HasValue ? PSFResultFlags.None : PSFResultFlags.IsNull; if (key.HasValue) { @@ -234,7 +236,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor if (phase == PSFExecutePhase.PostUpdate) { - // If not found, this is a new group added after the PreUpdate was done, so handle this as an insert. + // TODOtest: If not found, this is a new group added after the PreUpdate was done, so handle this as an insert. if (!changeTracker.FindGroup(this.Id, out groupOrdinal)) { phase = PSFExecutePhase.Insert; @@ -243,7 +245,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor { ref GroupKeysPair groupKeysPair = ref changeTracker.GetGroupRef(groupOrdinal); StoreKeys(ref groupKeysPair.After, keyMem, keyMemLen, flags, flagsMemLen); - this.MarkChanges(groupKeysPair); + this.MarkChanges(ref groupKeysPair); // TODOtest: In debug, for initial dev, follow chains to assert the values match what is in the record's compositeKey if (!groupKeysPair.HasChanges) return Status.OK; @@ -266,7 +268,8 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey, ref psfValue, ref input, lsn), PSFExecutePhase.PostUpdate => session.PsfUpdate(ref changeTracker.GetGroupRef(groupOrdinal), ref psfValue, ref input, lsn, changeTracker), - PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey, ref psfValue, ref input, lsn, changeTracker), + PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey, ref psfValue, ref input, lsn, + changeTracker), _ => throw new PSFInternalErrorException("Unknown PSF execution Phase {phase}") }; } @@ -333,7 +336,7 @@ public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key) var keyPtr = new PSFCompositeKey.PtrWrapper(this.keySize, this.bufferPool); keyPtr.Set(ref key); - // These indirections are necessary to avoid issues with passing ref or unsafe to a enumerator function. + // These indirections are necessary to avoid issues with passing ref or unsafe to an enumerator function. // TODOperf: PSFInput* and PSFOutput* are classes because we communicate through interfaces to avoid // having to add additional generic args. Interfaces on structs incur boxing overhead (plus the complexity // of mutable structs). But check the performance here; if necessary perhaps I can figure out a way to @@ -346,10 +349,10 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe PSFInputSecondary input) { // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them - // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr + // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr. Also consider + // having TPSFKeys[] for a single PSF walk through in parallel, so the FHT log memory access is sequential. var secondaryOutput = new PSFOutputSecondary(this.psfValueAccessor); - var readArgs = new PSFReadArgs, PSFValue>( - input, secondaryOutput); + var readArgs = new PSFReadArgs, PSFValue>(input, secondaryOutput); var session = this.GetSession(); HashSet deadRecs = null; diff --git a/cs/src/core/Index/PSF/PSFInput.cs b/cs/src/core/Index/PSF/PSFInput.cs index e52d95e2c..046969602 100644 --- a/cs/src/core/Index/PSF/PSFInput.cs +++ b/cs/src/core/Index/PSF/PSFInput.cs @@ -62,6 +62,8 @@ public interface IPSFInput bool EqualsAt(ref TKey queryKey, ref TKey storedKey); } + // TODO: Trim IPSFInput to only what PSFInputPrimaryReadAddress needs + /// /// Input to PsfInternalReadAddress on the primary (stores user values) FasterKV to retrieve the Key and Value /// for a logicalAddress returned from the secondary FasterKV instances. This class is FasterKV-provider-specific. @@ -136,12 +138,14 @@ internal PSFInputSecondary(int psfOrd, ICompositeKeyComparer this.resultFlags is null; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public long GetHashCode64At(ref PSFCompositeKey cKey) - => this.comparer.GetHashCode64(this.PsfOrdinal, ref cKey); + => this.comparer.GetHashCode64(this.IsQuery ? 0 : this.PsfOrdinal, ref cKey); [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool EqualsAt(ref PSFCompositeKey queryKey, ref PSFCompositeKey storedKey) - => this.comparer.Equals(this.resultFlags is null, this.PsfOrdinal, ref queryKey, ref storedKey); + => this.comparer.Equals(this.IsQuery, this.PsfOrdinal, ref queryKey, ref storedKey); } } diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index c0c37c6e3..d2bf8783e 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -92,6 +92,8 @@ private void VerifyIsBlittable() private void VerifyIsOurPSF(PSF psf) { + if (psf is null) + throw new PSFArgumentException($"The PSF cannot be null."); if (!this.psfNames.TryGetValue(psf.Name, out Guid id) || id != psf.Id) throw new PSFArgumentException($"The PSF {psf.Name} is not registered with this FasterKV."); } diff --git a/cs/src/core/Index/PSF/PSFValue.cs b/cs/src/core/Index/PSF/PSFValue.cs index 018805380..5db008947 100644 --- a/cs/src/core/Index/PSF/PSFValue.cs +++ b/cs/src/core/Index/PSF/PSFValue.cs @@ -3,6 +3,7 @@ using System; using System.Runtime.CompilerServices; +using System.Text; namespace FASTER.core { From b970991a48221b50288880c9f74afae2ce40b90d Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Wed, 17 Jun 2020 13:01:55 -0700 Subject: [PATCH 09/19] Change to KeyPointer --- cs/playground/FasterPSFSample/CombinedKey.cs | 2 +- .../FasterPSFSample/FasterPSFSample.cs | 26 +- .../Properties/launchSettings.json | 3 +- cs/src/core/Index/FASTER/FASTERImpl.cs | 52 +- cs/src/core/Index/FASTER/FASTERThread.cs | 2 +- cs/src/core/Index/PSF/CompositeKey.cs | 63 +++ .../Index/PSF/FasterPSFContextOperations.cs | 12 +- cs/src/core/Index/PSF/FasterPSFImpl.cs | 462 ++++++++---------- .../Index/PSF/FasterPSFSessionOperations.cs | 10 +- cs/src/core/Index/PSF/GroupKeys.cs | 1 - cs/src/core/Index/PSF/IPSFValueAccessor.cs | 20 - cs/src/core/Index/PSF/KeyAccessor.cs | 120 +++++ cs/src/core/Index/PSF/KeyPointer.cs | 36 ++ cs/src/core/Index/PSF/PSFChangeTracker.cs | 8 +- cs/src/core/Index/PSF/PSFCompositeKey.cs | 121 ----- cs/src/core/Index/PSF/PSFFunctions.cs | 35 +- cs/src/core/Index/PSF/PSFGroup.cs | 128 +++-- cs/src/core/Index/PSF/PSFInput.cs | 61 +-- cs/src/core/Index/PSF/PSFOutput.cs | 42 +- cs/src/core/Index/PSF/PSFValue.cs | 78 --- 20 files changed, 614 insertions(+), 668 deletions(-) create mode 100644 cs/src/core/Index/PSF/CompositeKey.cs delete mode 100644 cs/src/core/Index/PSF/IPSFValueAccessor.cs create mode 100644 cs/src/core/Index/PSF/KeyAccessor.cs create mode 100644 cs/src/core/Index/PSF/KeyPointer.cs delete mode 100644 cs/src/core/Index/PSF/PSFCompositeKey.cs delete mode 100644 cs/src/core/Index/PSF/PSFValue.cs diff --git a/cs/playground/FasterPSFSample/CombinedKey.cs b/cs/playground/FasterPSFSample/CombinedKey.cs index b1a32133a..8d93de87b 100644 --- a/cs/playground/FasterPSFSample/CombinedKey.cs +++ b/cs/playground/FasterPSFSample/CombinedKey.cs @@ -32,7 +32,7 @@ public CombinedKey(int countBin) public override string ToString() { var valueType = (Constants.ValueType)this.ValueType; - var valueTypeString = $"{nameof(Constants.ValueType)}.{valueType}"; + var valueTypeString = $"{valueType}"; return valueType switch { Constants.ValueType.Size => $"{valueTypeString}: {(Constants.Size)this.ValueInt}", diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs index 4930501c7..fce0286fd 100644 --- a/cs/playground/FasterPSFSample/FasterPSFSample.cs +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -9,6 +9,7 @@ using System.Diagnostics; using System.Drawing; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; namespace FasterPSFSample @@ -21,7 +22,8 @@ public partial class FasterPSFSample internal static Dictionary keyDict = new Dictionary(); - internal static List lastBinKeys = new List(); + internal static HashSet lastBinKeys = new HashSet(); + private static int initialSkippedLastBinCount; private static int nextId = 1000000000; @@ -120,10 +122,18 @@ internal static void RunUpserts(FPSF(FPSF fpsf) where TValue : IOrders, new() where TOutput : IOutput, new() @@ -255,6 +265,8 @@ FasterKVProviderData[] GetSizeDatas(Constants.Size size) // Reuse the same key session.Upsert(ref providerData.GetKey(), ref value, context, serialNo); + + RemoveIfSkippedLastBinKey(ref providerData.GetKey()); } ++serialNo; @@ -296,6 +308,7 @@ FasterKVProviderData[] GetColorDatas(Color color) { // This will call Functions<>.InPlaceUpdater. session.RMW(ref providerData.GetKey(), ref input, context, serialNo); + RemoveIfSkippedLastBinKey(ref providerData.GetKey()); } ++serialNo; @@ -326,11 +339,12 @@ FasterKVProviderData[] GetCountDatas(int bin) // First show we've nothing in the last bin, and get all in bin7. var lastBinDatas = GetCountDatas(CountBinKey.LastBin); - var ok = lastBinDatas.Length == 0; - WriteResult(isInitial: true, "LastBin", 0, lastBinDatas.Length); + int expectedLastBinCount = initialSkippedLastBinCount - lastBinKeys.Count(); + var ok = lastBinDatas.Length == expectedLastBinCount; + WriteResult(isInitial: true, "LastBin", expectedLastBinCount, lastBinDatas.Length); var bin7Datas = GetCountDatas(bin7); - ok &= lastBinDatas.Length == 0; + ok &= bin7Datas.Length == bin7Count; WriteResult(isInitial: true, "Bin7", bin7Count, bin7Datas.Length); Console.WriteLine($"{indent2}Changing all Bin7 to LastBin"); @@ -348,7 +362,7 @@ FasterKVProviderData[] GetCountDatas(int bin) } ++serialNo; - var expectedLastBinCount = bin7Datas.Length; + expectedLastBinCount += bin7Datas.Length; lastBinDatas = GetCountDatas(CountBinKey.LastBin); ok &= lastBinDatas.Length == expectedLastBinCount; WriteResult(isInitial: false, "LastBin", expectedLastBinCount, lastBinDatas.Length); diff --git a/cs/playground/FasterPSFSample/Properties/launchSettings.json b/cs/playground/FasterPSFSample/Properties/launchSettings.json index 560f42813..99c85823e 100644 --- a/cs/playground/FasterPSFSample/Properties/launchSettings.json +++ b/cs/playground/FasterPSFSample/Properties/launchSettings.json @@ -1,8 +1,7 @@ { "profiles": { "FasterPSFSample": { - "commandName": "Project", - "commandLineArgs": "--multiGroup" + "commandName": "Project" } } } \ No newline at end of file diff --git a/cs/src/core/Index/FASTER/FASTERImpl.cs b/cs/src/core/Index/FASTER/FASTERImpl.cs index afb2bff00..8811ddb04 100644 --- a/cs/src/core/Index/FASTER/FASTERImpl.cs +++ b/cs/src/core/Index/FASTER/FASTERImpl.cs @@ -4,6 +4,7 @@ #pragma warning disable 0162 #define CPR +using System; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Threading; @@ -220,9 +221,19 @@ internal OperationStatus InternalRead( #region Upsert Operation + #region PSF Utilities private FasterKVProviderData CreateProviderData(ref Key key, long physicalAddress) => new FasterKVProviderData(this.hlog, ref key, ref hlog.GetValue(physicalAddress)); + private unsafe static void GetAfterRecordId(PSFChangeTracker, long> changeTracker, + ref Value value) + { + // This indirection is needed because this is the primary FasterKV. + Debug.Assert(typeof(Value) == typeof(long)); + var recordId = changeTracker.AfterRecordId; + Buffer.MemoryCopy(Unsafe.AsPointer(ref recordId), Unsafe.AsPointer(ref value), sizeof(long), sizeof(long)); + } + private void SetBeforeData(PSFChangeTracker, long> changeTracker, ref Key key, long logicalAddress, long physicalAddress) { @@ -236,6 +247,7 @@ private void SetAfterData(PSFChangeTracker, lon changeTracker.AfterRecordId = logicalAddress; changeTracker.AfterData = CreateProviderData(ref key, physicalAddress); } + #endregion PSF Utilities /// /// Upsert operation. Replaces the value corresponding to 'key' with provided 'value', if one exists @@ -1846,6 +1858,7 @@ private bool TraceBackForKeyMatch( out long foundLogicalAddress, out long foundPhysicalAddress) { + Debug.Assert(this.psfKeyAccessor is null); foundLogicalAddress = fromLogicalAddress; while (foundLogicalAddress >= minOffset) { @@ -1854,40 +1867,10 @@ private bool TraceBackForKeyMatch( { return true; } - else - { - foundLogicalAddress = hlog.GetInfo(foundPhysicalAddress).PreviousAddress; - //This makes testing REALLY slow - //Debug.WriteLine("Tracing back"); - continue; - } - } - foundPhysicalAddress = Constants.kInvalidAddress; - return false; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TraceBackForKeyMatch( - ref Key key, - long fromLogicalAddress, - long minOffset, - out long foundLogicalAddress, - out long foundPhysicalAddress, - IPSFInput psfInput) - { - foundLogicalAddress = fromLogicalAddress; - while (foundLogicalAddress >= minOffset) - { - foundPhysicalAddress = hlog.GetPhysicalAddress(foundLogicalAddress); - if (psfInput.EqualsAt(ref key, ref hlog.GetKey(foundPhysicalAddress))) - return true; - - foundLogicalAddress = *(this.psfValueAccessor.GetChainLinkPtrs(ref hlog.GetValue(foundPhysicalAddress)) - + psfInput.PsfOrdinal); + foundLogicalAddress = hlog.GetInfo(foundPhysicalAddress).PreviousAddress; //This makes testing REALLY slow //Debug.WriteLine("Tracing back"); - continue; } foundPhysicalAddress = Constants.kInvalidAddress; return false; @@ -2087,8 +2070,7 @@ private long TraceBackForOtherChainStart(long logicalAddress, int bit) #endregion #region Read Cache - private bool ReadFromCache(ref Key key, ref long logicalAddress, ref long physicalAddress, - ref int latestRecordVersion, IPSFInput psfInput = null) + private bool ReadFromCache(ref Key key, ref long logicalAddress, ref long physicalAddress, ref int latestRecordVersion) { HashBucketEntry entry = default; entry.word = logicalAddress; @@ -2101,9 +2083,7 @@ private bool ReadFromCache(ref Key key, ref long logicalAddress, ref long physic { if (!readcache.GetInfo(physicalAddress).Invalid) { - if (psfInput is null - ? comparer.Equals(ref key, ref readcache.GetKey(physicalAddress)) - : psfInput.EqualsAt(ref key, ref readcache.GetKey(physicalAddress))) + if (comparer.Equals(ref key, ref readcache.GetKey(physicalAddress))) { if ((logicalAddress & ~Constants.kReadCacheBitMask) >= readcache.SafeReadOnlyAddress) return true; diff --git a/cs/src/core/Index/FASTER/FASTERThread.cs b/cs/src/core/Index/FASTER/FASTERThread.cs index e393ec49c..65c185375 100644 --- a/cs/src/core/Index/FASTER/FASTERThread.cs +++ b/cs/src/core/Index/FASTER/FASTERThread.cs @@ -262,7 +262,7 @@ internal void InternalCompleteRetryRequest(FasterExecutionContext opCtx, FasterE if (pendingContext.psfUpdateArgs.ChangeTracker.FindGroup(psfInput.GroupId, out var ordinal)) { ref GroupKeysPair groupKeysPair = ref pendingContext.psfUpdateArgs.ChangeTracker.GetGroupRef(ordinal); - this.psfValueAccessor.SetRecordId(ref value, pendingContext.psfUpdateArgs.ChangeTracker.AfterRecordId); + GetAfterRecordId(pendingContext.psfUpdateArgs.ChangeTracker, ref value); var pcontext = default(PendingContext); PsfRcuInsert(groupKeysPair.After, ref value, ref pendingContext.input, ref pcontext, currentCtx, pendingContext.serialNum + 1); diff --git a/cs/src/core/Index/PSF/CompositeKey.cs b/cs/src/core/Index/PSF/CompositeKey.cs new file mode 100644 index 000000000..0e51a4604 --- /dev/null +++ b/cs/src/core/Index/PSF/CompositeKey.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + /// + /// Wraps the set of TPSFKeys for a record in the secondary FasterKV instance. + /// + /// + public unsafe struct CompositeKey + where TPSFKey : new() + { + // This class is essentially a "reinterpret_cast*>" implementation; there are no data members. + + /// + /// Get a reference to the key for the PSF identified by psfOrdinal. + /// + /// The ordinal of the PSF in its parent PSFGroup + /// Size of the KeyPointer{TPSFKey} struct + /// A reference to the key for the PSF identified by psfOrdinal. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ref KeyPointer GetKeyPointerRef(int psfOrdinal, int keyPointerSize) + => ref Unsafe.AsRef>((byte*)Unsafe.AsPointer(ref this) + keyPointerSize * psfOrdinal); + + /// + /// Get a reference to the key for the PSF identified by psfOrdinal. + /// + /// The ordinal of the PSF in its parent PSFGroup + /// Size of the KeyPointer{TPSFKey} struct + /// A reference to the key for the PSF identified by psfOrdinal. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ref TPSFKey GetKeyRef(int psfOrdinal, int keyPointerSize) + => ref GetKeyPointerRef(psfOrdinal, keyPointerSize).Key; + + internal class VarLenLength : IVariableLengthStruct> + { + private readonly int size; + + internal VarLenLength(int keyPointerSize, int psfCount) => this.size = keyPointerSize * psfCount; + + public int GetAverageLength() => this.size; + + public int GetInitialLength(ref Input _) => this.size; + + public int GetLength(ref CompositeKey _) => this.size; + } + + /// + /// This is the unused key comparer passed to the secondary FasterKV + /// + internal class UnusedKeyComparer : IFasterEqualityComparer> + { + public long GetHashCode64(ref CompositeKey cKey) + => throw new PSFInternalErrorException("Must use KeyAccessor instead"); + + public bool Equals(ref CompositeKey cKey1, ref CompositeKey cKey2) + => throw new PSFInternalErrorException("Must use KeyAccessor instead"); + } + } +} diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs index e13a66ce6..4bcbb25a7 100644 --- a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -56,10 +56,9 @@ internal Status ContextPsfInsert(ref Key key, ref Value value, ref Input input, } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfUpdate(ref GroupKeysPair groupKeysPair, ref Value value, ref Input input, long serialNo, + internal Status ContextPsfUpdate(ref GroupKeysPair groupKeysPair, ref Value value, ref Input input, long serialNo, FasterExecutionContext sessionCtx, - PSFChangeTracker changeTracker) - where TRecordId : struct + PSFChangeTracker changeTracker) { var pcontext = default(PendingContext); var psfInput = (IPSFInput)input; @@ -78,7 +77,7 @@ internal Status ContextPsfUpdate(ref GroupKeysPair gro if (status == Status.OK) { - this.psfValueAccessor.SetRecordId(ref value, changeTracker.AfterRecordId); + value = changeTracker.AfterRecordId; return PsfRcuInsert(groupKeysPair.After, ref value, ref input, ref pcontext, sessionCtx, serialNo + 1); } return status; @@ -98,10 +97,9 @@ private Status PsfRcuInsert(GroupKeys groupKeys, ref Value value, ref Input inpu } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfDelete(ref Key key, ref Value value, ref Input input, long serialNo, + internal Status ContextPsfDelete(ref Key key, ref Value value, ref Input input, long serialNo, FasterExecutionContext sessionCtx, - PSFChangeTracker changeTracker) - where TRecordId : struct + PSFChangeTracker changeTracker) { var pcontext = default(PendingContext); diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index 5343e69a8..e8c5c2a26 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -20,23 +20,58 @@ public unsafe partial class FasterKV { - internal IPSFValueAccessor psfValueAccessor; + internal IKeyAccessor psfKeyAccessor; + + bool ScanQueryChain(ref long logicalAddress, ref Key queryKey, ref int latestRecordVersion) + { + long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + var recordAddress = this.psfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + if (latestRecordVersion == -1) + latestRecordVersion = hlog.GetInfo(recordAddress).Version; + + while (true) + { + if (this.psfKeyAccessor.Equals(ref queryKey, physicalAddress)) + { + PsfTrace($" / {logicalAddress}"); + return true; + } + logicalAddress = this.psfKeyAccessor.GetPrevAddress(physicalAddress); + if (logicalAddress < hlog.HeadAddress) + break; // RECORD_ON_DISK or not found + physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + } + PsfTrace($"/{logicalAddress}"); + return false; + } + + [Conditional("PSF_TRACE")] + private void PsfTrace(string message) + { + if (!(this.psfKeyAccessor is null)) Console.Write(message); + } + + [Conditional("PSF_TRACE")] + private void PsfTraceLine(string message = null) + { + if (!(this.psfKeyAccessor is null)) Console.WriteLine(message ?? string.Empty); + } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal OperationStatus PsfInternalReadKey( ref Key queryKey, ref PSFReadArgs psfArgs, ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) { + // Note: This function is called only for the secondary FasterKV. var bucket = default(HashBucket*); var slot = default(int); - var physicalAddress = default(long); var latestRecordVersion = -1; var heldOperation = LatchOperation.None; var psfInput = psfArgs.Input; var psfOutput = psfArgs.Output; - var hash = psfInput.GetHashCode64At(ref queryKey); + var hash = this.psfKeyAccessor.GetHashCode64(ref queryKey, 0); // the queryKey has only one key var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); if (sessionCtx.phase != Phase.REST) @@ -46,12 +81,17 @@ internal OperationStatus PsfInternalReadKey( HashBucketEntry entry = default; var tagExists = FindTag(hash, tag, ref bucket, ref slot, ref entry); OperationStatus status; - long logicalAddress; + + // For PSFs, the addresses stored in the hash table point to KeyPointer entries, not the record header. + PsfTrace($"ReadKey: {this.psfKeyAccessor?.GetString(ref queryKey, 0)} | hash {hash} |"); + long logicalAddress = Constants.kInvalidAddress; if (tagExists) { logicalAddress = entry.Address; + PsfTrace($" {logicalAddress}"); - if (UseReadCache && ReadFromCache(ref queryKey, ref logicalAddress, ref physicalAddress, ref latestRecordVersion, psfInput)) +#if false // TODO: Move from the LogicalAddress to the record header for ReadFromCache + if (UseReadCache && ReadFromCache(ref queryKey, ref logicalAddress, ref physicalAddress, ref latestRecordVersion, psfInput)) { if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) { @@ -62,55 +102,42 @@ internal OperationStatus PsfInternalReadKey( ref readcache.GetValue(physicalAddress), hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: false).Status; } +#endif if (logicalAddress >= hlog.HeadAddress) { - physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - if (latestRecordVersion == -1) - latestRecordVersion = hlog.GetInfo(physicalAddress).Version; - - if (!psfInput.EqualsAt(ref queryKey, ref hlog.GetKey(physicalAddress))) - { - logicalAddress = *(this.psfValueAccessor.GetChainLinkPtrs(ref hlog.GetValue(physicalAddress)) + psfInput.PsfOrdinal); - if (logicalAddress > hlog.BeginAddress) - TraceBackForKeyMatch(ref queryKey, - logicalAddress, - hlog.HeadAddress, - out logicalAddress, - out physicalAddress, - psfInput); - } + if (!ScanQueryChain(ref logicalAddress, ref psfInput.QueryKeyRef, ref latestRecordVersion)) + goto ProcessAddress; // RECORD_ON_DISK or not found } } else { - // no tag found - return OperationStatus.NOTFOUND; + PsfTraceLine($" 0"); + return OperationStatus.NOTFOUND; // no tag found } - #endregion +#endregion if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) { + PsfTraceLine("CPR_SHIFT_DETECTED"); status = OperationStatus.CPR_SHIFT_DETECTED; goto CreatePendingContext; // Pivot thread } - #region Normal processing + #region Normal processing - // Mutable region (even fuzzy region is included here) - if (logicalAddress >= hlog.SafeReadOnlyAddress) - { - return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), - ref hlog.GetValue(physicalAddress), - hlog.GetInfo(physicalAddress).Tombstone, isConcurrent:true).Status; - } - - // Immutable region - else if (logicalAddress >= hlog.HeadAddress) + ProcessAddress: + PsfTraceLine(); + if (logicalAddress >= hlog.HeadAddress) { - return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), - ref hlog.GetValue(physicalAddress), - hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: true).Status; + // Mutable region (even fuzzy region is included here) is above SafeReadOnlyAddress and + // is concurrent; Immutable region will not be changed. + long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + long recordAddress = this.psfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + return psfOutput.Visit(psfInput.PsfOrdinal, physicalAddress, + ref hlog.GetValue(recordAddress), + hlog.GetInfo(recordAddress).Tombstone, + isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; } // On-Disk Region @@ -140,9 +167,9 @@ ref hlog.GetValue(physicalAddress), return OperationStatus.NOTFOUND; } - #endregion +#endregion - #region Create pending context +#region Create pending context CreatePendingContext: { pendingContext.type = OperationType.PSF_READ_KEY; @@ -151,13 +178,14 @@ ref hlog.GetValue(physicalAddress), pendingContext.output = default; pendingContext.userContext = default; pendingContext.entry.word = entry.word; - pendingContext.logicalAddress = logicalAddress; + pendingContext.logicalAddress = // TODO: fix this in the read callback + this.psfKeyAccessor.GetRecordAddressFromKeyLogicalAddress(logicalAddress, psfInput.PsfOrdinal); pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; pendingContext.heldLatch = heldOperation; pendingContext.psfReadArgs = psfArgs; } - #endregion +#endregion return status; } @@ -167,7 +195,7 @@ internal OperationStatus PsfInternalReadAddress( ref PSFReadArgs psfArgs, ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) { - var physicalAddress = default(long); + // Note: This function is called for both the primary and secondary FasterKV. var latestRecordVersion = -1; var psfInput = psfArgs.Input; @@ -175,8 +203,12 @@ internal OperationStatus PsfInternalReadAddress( OperationStatus status; - #region Look up record in in-memory HybridLog +#region Look up record in in-memory HybridLog + // For PSFs, the addresses stored in the hash table point to KeyPointer entries, not the record header. long logicalAddress = psfInput.ReadLogicalAddress; + PsfTrace($" ReadAddr: | {logicalAddress}"); + +#if false // TODO: Move from the LogicalAddress to the record header for ReadFromCache if (UseReadCache && ReadFromCache(ref logicalAddress, ref physicalAddress, ref latestRecordVersion)) { if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) @@ -188,37 +220,49 @@ internal OperationStatus PsfInternalReadAddress( ref readcache.GetValue(physicalAddress), hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: false).Status; } +#endif if (logicalAddress >= hlog.HeadAddress) { - physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - if (latestRecordVersion == -1) - latestRecordVersion = hlog.GetInfo(physicalAddress).Version; + if (this.psfKeyAccessor is null) + { + if (latestRecordVersion == -1) + latestRecordVersion = hlog.GetInfo(hlog.GetPhysicalAddress(logicalAddress)).Version; + } + else if (!ScanQueryChain(ref logicalAddress, ref psfInput.QueryKeyRef, ref latestRecordVersion)) + { + goto ProcessAddress; // RECORD_ON_DISK or not found + } } - #endregion +#endregion if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) { + PsfTraceLine("CPR_SHIFT_DETECTED"); status = OperationStatus.CPR_SHIFT_DETECTED; goto CreatePendingContext; // Pivot thread } - #region Normal processing +#region Normal processing - // Mutable region (even fuzzy region is included here) - if (logicalAddress >= hlog.SafeReadOnlyAddress) - { - return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), - ref hlog.GetValue(physicalAddress), - hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: true).Status; - } - - // Immutable region - else if (logicalAddress >= hlog.HeadAddress) + ProcessAddress: + PsfTraceLine(); + if (logicalAddress >= hlog.HeadAddress) { - return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + // Mutable region (even fuzzy region is included here) is above SafeReadOnlyAddress and + // is concurrent; Immutable region will not be changed. + long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + if (this.psfKeyAccessor is null) + return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), ref hlog.GetValue(physicalAddress), - hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: true).Status; + hlog.GetInfo(physicalAddress).Tombstone, + isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; + + long recordAddress = this.psfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + return psfOutput.Visit(psfInput.PsfOrdinal, physicalAddress, + ref hlog.GetValue(recordAddress), + hlog.GetInfo(recordAddress).Tombstone, + isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; } // On-Disk Region @@ -226,9 +270,8 @@ ref hlog.GetValue(physicalAddress), { // We do not have a key here, so we cannot get the hash, latch, etc. for CPR and must retry later; // this is the only Read operation that goes through Retry rather than pending. TODOtest: Retry - status = sessionCtx.phase == Phase.PREPARE - ? OperationStatus.RETRY_LATER - : OperationStatus.RECORD_ON_DISK; + // TODO: Now we have the psfInput.QueryKeyRef so we could get the hashcode; revisit this + status = sessionCtx.phase == Phase.PREPARE ? OperationStatus.RETRY_LATER : OperationStatus.RECORD_ON_DISK; goto CreatePendingContext; } else @@ -239,7 +282,7 @@ ref hlog.GetValue(physicalAddress), #endregion - #region Create pending context +#region Create pending context CreatePendingContext: { pendingContext.type = OperationType.PSF_READ_ADDRESS; @@ -248,7 +291,9 @@ ref hlog.GetValue(physicalAddress), pendingContext.output = default; pendingContext.userContext = default; pendingContext.entry.word = default; - pendingContext.logicalAddress = logicalAddress; + pendingContext.logicalAddress = this.psfKeyAccessor is null // TODO: fix this in the read callback + ? logicalAddress + : this.psfKeyAccessor.GetRecordAddressFromKeyLogicalAddress(logicalAddress, psfInput.PsfOrdinal); pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; pendingContext.heldLatch = LatchOperation.None; @@ -259,9 +304,13 @@ ref hlog.GetValue(physicalAddress), return status; } - [Conditional("PSF_TRACE")] static void PsfTrace(string message) => Console.Write(message); - - [Conditional("PSF_TRACE")] static void PsfTraceLine(string message) => Console.WriteLine(message); + unsafe struct CASHelper + { + internal HashBucket* bucket; + internal HashBucketEntry entry; + internal long hash; + internal int slot; + } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal OperationStatus PsfInternalInsert( @@ -269,228 +318,163 @@ internal OperationStatus PsfInternalInsert( ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) { var status = default(OperationStatus); - var bucket = default(HashBucket*); - var slot = default(int); - var logicalAddress = Constants.kInvalidAddress; - var physicalAddress = default(long); - var latchOperation = default(LatchOperation); var latestRecordVersion = -1; - var entry = default(HashBucketEntry); var psfInput = input as IPSFInput; - // Update the PSFValue links for chains with IsNullAt false (indicating a match with the + // Update the KeyPointer links for chains with IsNullAt false (indicating a match with the // corresponding PSF) to point to the previous records for all keys in the composite key. - // Note: We're not checking for a previous occurrence of the PSFValue's recordId because + // Note: We're not checking for a previous occurrence of the input value (the recordId) because // we are doing insert only here; the update part of upsert is done in PsfInternalUpdate. - var psfCount = this.psfValueAccessor.PSFCount; - long* hashes = stackalloc long[psfCount]; - long* chainLinkPtrs = this.psfValueAccessor.GetChainLinkPtrs(ref value); - PsfTrace($" {value} |"); + // TODO: Limit size of stackalloc based on # of PSFs. + var psfCount = this.psfKeyAccessor.KeyCount; + CASHelper* casHelpers = stackalloc CASHelper[psfCount]; + PsfTrace($"Insert: {this.psfKeyAccessor.GetString(ref compositeKey)} | rId {value} |"); for (psfInput.PsfOrdinal = 0; psfInput.PsfOrdinal < psfCount; ++psfInput.PsfOrdinal) { // For RCU, or in case we had to retry due to CPR_SHIFT and somehow managed to delete // the previously found record, clear out the chain link pointer. - long* chainLinkPtr = chainLinkPtrs + psfInput.PsfOrdinal; - *chainLinkPtr = Constants.kInvalidAddress; + this.psfKeyAccessor.SetPrevAddress(ref compositeKey, psfInput.PsfOrdinal, Constants.kInvalidAddress); if (psfInput.IsNullAt) { - PsfTrace($" -0-"); + PsfTrace($" null"); continue; } - var hash = psfInput.GetHashCode64At(ref compositeKey); - *(hashes + psfInput.PsfOrdinal) = hash; - var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); + ref CASHelper casHelper = ref casHelpers[psfInput.PsfOrdinal]; + casHelper.hash = this.psfKeyAccessor.GetHashCode64(ref compositeKey, psfInput.PsfOrdinal); + var tag = (ushort)((ulong)casHelper.hash >> Constants.kHashTagShift); if (sessionCtx.phase != Phase.REST) - HeavyEnter(hash, sessionCtx); + HeavyEnter(casHelper.hash, sessionCtx); -#region Trace back for record in in-memory HybridLog - entry = default; - if (FindTag(hash, tag, ref bucket, ref slot, ref entry)) +#region Look up record in in-memory HybridLog + FindOrCreateTag(casHelper.hash, tag, ref casHelper.bucket, ref casHelper.slot, ref casHelper.entry, hlog.BeginAddress); + + // For PSFs, the addresses stored in the hash table point to KeyPointer entries, not the record header. + var logicalAddress = casHelper.entry.Address; + if (logicalAddress >= hlog.BeginAddress) { - logicalAddress = entry.Address; - PsfTrace($" {logicalAddress}/"); - - // TODOtest: If this fails for any TPSFKey in the composite key, we'll create the pending - // context and come back here on the retry and overwrite any previously-obtained logicalAddress - // at the chainLinkPtr. - if (UseReadCache && ReadFromCache(ref compositeKey, ref logicalAddress, ref physicalAddress, - ref latestRecordVersion, psfInput)) - { - if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) - { - status = OperationStatus.CPR_SHIFT_DETECTED; - goto CreatePendingContext; // Pivot thread - } - } + PsfTrace($" {logicalAddress}"); + + if (logicalAddress < hlog.BeginAddress) + continue; if (logicalAddress >= hlog.HeadAddress) { - physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - if (latestRecordVersion == -1) - latestRecordVersion = hlog.GetInfo(physicalAddress).Version; - - if (!psfInput.EqualsAt(ref compositeKey, ref hlog.GetKey(physicalAddress))) + // Note that we do not backtrace here because we are not replacing the value at the key; + // instead, we insert at the top of the hash chain. Track the latest record version we've seen. + long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + var recordAddress = this.psfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + if (hlog.GetInfo(physicalAddress).Tombstone) { - logicalAddress = *(this.psfValueAccessor.GetChainLinkPtrs(ref hlog.GetValue(physicalAddress)) + psfInput.PsfOrdinal); - if (logicalAddress > hlog.BeginAddress) - TraceBackForKeyMatch(ref compositeKey, - logicalAddress, - hlog.HeadAddress, - out logicalAddress, - out physicalAddress, - psfInput); + // The chain might extend past a tombstoned record so we must include it in the chain + // unless its prevLink at psfOrdinal is invalid. + var prevAddress = this.psfKeyAccessor.GetPrevAddress(physicalAddress); + if (prevAddress < hlog.BeginAddress) + continue; } + latestRecordVersion = Math.Max(latestRecordVersion, hlog.GetInfo(recordAddress).Version); } - PsfTrace($"{logicalAddress}"); - if (logicalAddress < hlog.BeginAddress) - continue; - - if (hlog.GetInfo(physicalAddress).Tombstone) - { - // The chain might extend past a tombstoned record so we must include it in the chain - // unless its prevLink at psfOrdinal is invalid. - long* prevLinks = this.psfValueAccessor.GetChainLinkPtrs(ref hlog.GetValue(physicalAddress)); - long* prevLink = prevLinks + psfInput.PsfOrdinal; - if (*prevLink < hlog.BeginAddress) - continue; - } - *chainLinkPtr = logicalAddress; + this.psfKeyAccessor.SetPrevAddress(ref compositeKey, psfInput.PsfOrdinal, logicalAddress); } else { - PsfTrace($" {logicalAddress}/{logicalAddress}"); + PsfTrace($" 0"); } #endregion } - #region Entry latch operation - if (sessionCtx.phase != Phase.REST) +#region Entry latch operation + // No actual checkpoint locking will be done because this is Insert; only the current thread can write to + // the record we're about to create, and no readers can see it until it is successfully inserted. However, we + // must pivot and retry any insertions if we have seen a later version in any record in the hash table. + if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) { - switch (sessionCtx.phase) - { - case Phase.PREPARE: - { - if (HashBucket.TryAcquireSharedLatch(bucket)) - { - // Set to release shared latch (default) - latchOperation = LatchOperation.Shared; - if (latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) - { - status = OperationStatus.CPR_SHIFT_DETECTED; - goto CreatePendingContext; // Pivot Thread - } - break; // Normal Processing - } - else - { - status = OperationStatus.CPR_SHIFT_DETECTED; - goto CreatePendingContext; // Pivot Thread - } - } - case Phase.IN_PROGRESS: - { - if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) - { - if (HashBucket.TryAcquireExclusiveLatch(bucket)) - { - // Set to release exclusive latch (default) - latchOperation = LatchOperation.Exclusive; - goto CreateNewRecord; // Create a (v+1) record - } - else - { - status = OperationStatus.RETRY_LATER; - goto CreatePendingContext; // Go Pending - } - } - break; // Normal Processing - } - case Phase.WAIT_PENDING: - { - if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) - { - if (HashBucket.NoSharedLatches(bucket)) - { - goto CreateNewRecord; // Create a (v+1) record - } - else - { - status = OperationStatus.RETRY_LATER; - goto CreatePendingContext; // Go Pending - } - } - break; // Normal Processing - } - case Phase.WAIT_FLUSH: - { - if (latestRecordVersion != -1 && latestRecordVersion < sessionCtx.version) - { - goto CreateNewRecord; // Create a (v+1) record - } - break; // Normal Processing - } - default: - break; - } + PsfTraceLine("CPR_SHIFT_DETECTED"); + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot Thread } -#endregion - Debug.Assert(latestRecordVersion <= sessionCtx.version); + goto CreateNewRecord; +#endregion #region Create new record in the mutable region CreateNewRecord: { - // Immutable region or new record + // Create the new record. Because we are updating multiple hash buckets, mark the + // record as invalid to start, so it is not visible until we have successfully + // updated all chains. var recordSize = hlog.GetRecordSize(ref compositeKey, ref value); BlockAllocate(recordSize, out long newLogicalAddress, sessionCtx); var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); RecordInfo.WriteInfo(ref hlog.GetInfo(newPhysicalAddress), sessionCtx.version, - final:true, tombstone: psfInput.IsDelete, invalidBit:false, - Constants.kInvalidAddress); // We manage all prev addresses within PSFValue - hlog.ShallowCopy(ref compositeKey, ref hlog.GetKey(newPhysicalAddress)); + final:true, tombstone: psfInput.IsDelete, invalidBit:true, + Constants.kInvalidAddress); // We manage all prev addresses within CompositeKey + ref Key storedKey = ref hlog.GetKey(newPhysicalAddress); + hlog.ShallowCopy(ref compositeKey, ref storedKey); functions.SingleWriter(ref compositeKey, ref value, ref hlog.GetValue(newPhysicalAddress)); - PsfTraceLine($" | {newLogicalAddress}"); - for (var psfOrdinal = 0; psfOrdinal < psfCount; ++psfOrdinal) + PsfTraceLine(); + newLogicalAddress += RecordInfo.GetLength(); + for (psfInput.PsfOrdinal = 0; psfInput.PsfOrdinal < psfCount; + ++psfInput.PsfOrdinal, newLogicalAddress += this.psfKeyAccessor.KeyPointerSize) { - psfInput.PsfOrdinal = psfOrdinal; + var casHelper = casHelpers[psfInput.PsfOrdinal]; + var tag = (ushort)((ulong)casHelper.hash >> Constants.kHashTagShift); + + PsfTrace($" ({psfInput.PsfOrdinal}): {casHelper.hash} {tag} | newLA {newLogicalAddress} | prev {casHelper.entry.word}"); if (psfInput.IsNullAt) + { + PsfTraceLine(" null"); continue; + } - var hash = *(hashes + psfOrdinal); - var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); - entry = default; - FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry, hlog.BeginAddress); - - var updatedEntry = default(HashBucketEntry); - updatedEntry.Tag = tag; - updatedEntry.Address = newLogicalAddress & Constants.kAddressMask; - updatedEntry.Pending = entry.Pending; - updatedEntry.Tentative = false; + var newEntry = default(HashBucketEntry); + newEntry.Tag = tag; + newEntry.Address = newLogicalAddress & Constants.kAddressMask; + newEntry.Pending = casHelper.entry.Pending; + newEntry.Tentative = false; var foundEntry = default(HashBucketEntry); - foundEntry.word = Interlocked.CompareExchange(ref bucket->bucket_entries[slot], - updatedEntry.word, entry.word); - - if (foundEntry.word != entry.word) + while (true) { - hlog.GetInfo(newPhysicalAddress).Invalid = true; + // If we do not succeed on the exchange, another thread has updated the slot, or we have done so + // with a colliding hash value from earlier in the current record. As long as we satisfy the + // invariant that the chain points downward (to lower addresses), we can retry. + foundEntry.word = Interlocked.CompareExchange(ref casHelper.bucket->bucket_entries[casHelper.slot], + newEntry.word, casHelper.entry.word); + if (foundEntry.word == casHelper.entry.word) + break; + + if (foundEntry.word < newEntry.word) + { + PsfTrace($" / {foundEntry.Address}"); + casHelper.entry.word = foundEntry.word; + this.psfKeyAccessor.SetPrevAddress(ref storedKey, psfInput.PsfOrdinal, foundEntry.Address); + continue; + } + + // We can't satisfy the always-downward invariant, so leave the record marked Invalid and go + // around again to try inserting another record. + PsfTraceLine("RETRY_NOW"); status = OperationStatus.RETRY_NOW; goto LatchRelease; } + + // Success + PsfTraceLine(" ins"); + hlog.GetInfo(newPhysicalAddress).Invalid = false; } status = OperationStatus.SUCCESS; goto LatchRelease; } - #endregion +#endregion - #region Create pending context +#region Create pending context CreatePendingContext: { psfInput.PsfOrdinal = Constants.kInvalidPsfOrdinal; @@ -500,29 +484,17 @@ internal OperationStatus PsfInternalInsert( pendingContext.value = hlog.GetValueContainer(ref value); pendingContext.input = input; pendingContext.userContext = default; - pendingContext.entry.word = entry.word; - pendingContext.logicalAddress = logicalAddress; + pendingContext.entry.word = default; + pendingContext.logicalAddress = Constants.kInvalidAddress; pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; } - #endregion +#endregion - #region Latch release +#region Latch release LatchRelease: - { - switch (latchOperation) - { - case LatchOperation.Shared: - HashBucket.ReleaseSharedLatch(bucket); - break; - case LatchOperation.Exclusive: - HashBucket.ReleaseExclusiveLatch(bucket); - break; - default: - break; - } - } - #endregion + // No actual latching was done. +#endregion return status == OperationStatus.RETRY_NOW ? PsfInternalInsert(ref compositeKey, ref value, ref input, ref pendingContext, sessionCtx, lsn) diff --git a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs index 4ece8fa23..74055cce6 100644 --- a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs @@ -56,9 +56,8 @@ internal Status PsfReadAddress(ref PSFReadArgs psfArgs, long serialN } } - internal Status PsfUpdate(ref GroupKeysPair groupKeysPair, ref Value value, ref Input input, long serialNo, - PSFChangeTracker changeTracker) - where TRecordId : struct + internal Status PsfUpdate(ref GroupKeysPair groupKeysPair, ref Value value, ref Input input, long serialNo, + PSFChangeTracker changeTracker) { // Called on the secondary FasterKV if (SupportAsync) UnsafeResumeThread(); @@ -72,9 +71,8 @@ internal Status PsfUpdate(ref GroupKeysPair groupKeysP } } - internal Status PsfDelete(ref Key key, ref Value value, ref Input input, long serialNo, - PSFChangeTracker changeTracker) - where TRecordId : struct + internal Status PsfDelete(ref Key key, ref Value value, ref Input input, long serialNo, + PSFChangeTracker changeTracker) { // Called on the secondary FasterKV if (SupportAsync) UnsafeResumeThread(); diff --git a/cs/src/core/Index/PSF/GroupKeys.cs b/cs/src/core/Index/PSF/GroupKeys.cs index ab06d3974..3805ae0fc 100644 --- a/cs/src/core/Index/PSF/GroupKeys.cs +++ b/cs/src/core/Index/PSF/GroupKeys.cs @@ -40,7 +40,6 @@ public void Dispose() internal struct GroupKeysPair : IDisposable { internal long GroupId; - internal int KeySize; // If the PSFGroup found the RecordId in its IPUCache, we carry it here. internal long LogicalAddress; diff --git a/cs/src/core/Index/PSF/IPSFValueAccessor.cs b/cs/src/core/Index/PSF/IPSFValueAccessor.cs deleted file mode 100644 index 89c9efabe..000000000 --- a/cs/src/core/Index/PSF/IPSFValueAccessor.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace FASTER.core -{ - /// - /// This interface is a generic indirection to access the pieces of a PSFValue{TPSFKey}. - /// - /// - internal unsafe interface IPSFValueAccessor - { - int PSFCount { get; } - - int RecordIdSize { get; } - - long* GetChainLinkPtrs(ref TPSFValue value); - - void SetRecordId(ref TPSFValue value, TRecordId recordId); - } -} \ No newline at end of file diff --git a/cs/src/core/Index/PSF/KeyAccessor.cs b/cs/src/core/Index/PSF/KeyAccessor.cs new file mode 100644 index 000000000..211670b6b --- /dev/null +++ b/cs/src/core/Index/PSF/KeyAccessor.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Runtime.CompilerServices; +using System.Text; + +namespace FASTER.core +{ + /// + /// Internal interface to bridge the generic definition for composite key type. + /// + /// + internal interface IKeyAccessor + { + int KeyCount { get; } + + int KeyPointerSize { get; } + + long GetPrevAddress(long physicalAddress); + + void SetPrevAddress(ref TCompositeKey key, int psfOrdinal, long prevAddress); + + long GetRecordAddressFromKeyPhysicalAddress(long physicalAddress); + + long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, long psfOrdinal); + + long GetHashCode64(ref TCompositeKey key, int psfOrdinal); + + bool Equals(ref TCompositeKey queryKey, long physicalAddress); + + string GetString(ref TCompositeKey key, int psfOrdinal = -1); + } + + /// + /// Provides access to the internals that are hidden behind + /// the Key typeparam of the secondary FasterKV. + /// + /// The type of the Key returned by a PSF function + internal unsafe class KeyAccessor : IKeyAccessor> + where TPSFKey : new() + { + private readonly IFasterEqualityComparer userComparer; + + internal KeyAccessor(IFasterEqualityComparer userComparer, int keyCount, int keyPointerSize) + { + this.userComparer = userComparer; + this.KeyCount = keyCount; + this.KeyPointerSize = keyPointerSize; + } + + public int KeyCount { get; } + + public int KeyPointerSize { get; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetPrevAddress(long physicalAddress) + => GetKeyPointerRef(physicalAddress).PrevAddress; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SetPrevAddress(ref CompositeKey key, int psfOrdinal, long prevAddress) + => this.GetKeyPointerRef(ref key, psfOrdinal).PrevAddress = prevAddress; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetRecordAddressFromKeyPhysicalAddress(long physicalAddress) + => physicalAddress - this.GetKeyPointerRef(physicalAddress).PsfOrdinal * this.KeyPointerSize - RecordInfo.GetLength(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, long psfOrdinal) + => logicalAddress - psfOrdinal* this.KeyPointerSize - RecordInfo.GetLength(); // TODO: Assumes all PSFs are present + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetHashCode64(ref CompositeKey key, int psfOrdinal) + { + // TODO: Incorporate psfOrdinal into the hashcode; remember that queryKey has only position 0 though. + ref KeyPointer queryKeyPointer = ref key.GetKeyPointerRef(psfOrdinal, this.KeyPointerSize); + return this.userComparer.GetHashCode64(ref queryKeyPointer.Key); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool Equals(ref CompositeKey queryKey, long physicalAddress) + { + // The query key only has a single key in it--the one we're trying to match. + ref KeyPointer queryKeyPointer = ref queryKey.GetKeyPointerRef(0, this.KeyPointerSize); + ref KeyPointer storedKeyPointer = ref GetKeyPointerRef(physicalAddress); + return queryKeyPointer.PsfOrdinal == storedKeyPointer.PsfOrdinal && + this.userComparer.Equals(ref queryKeyPointer.Key, ref storedKeyPointer.Key); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ref KeyPointer GetKeyPointerRef(ref CompositeKey key, int psfOrdinal) + => ref key.GetKeyPointerRef(psfOrdinal, this.KeyPointerSize); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal unsafe ref KeyPointer GetKeyPointerRef(long physicalAddress) + => ref Unsafe.AsRef>((byte*)physicalAddress); + + public string GetString(ref CompositeKey key, int psfOrdinal = -1) + { + var sb = new StringBuilder("{"); + + if (psfOrdinal == -1) + { + for (var ii = 0; ii < this.KeyCount; ++ii) + { + if (ii > 0) + sb.Append(", "); + ref KeyPointer keyPointer = ref this.GetKeyPointerRef(ref key, ii); + sb.Append(keyPointer.IsNull ? "null" : keyPointer.Key.ToString()); + } + } + else + { + ref KeyPointer keyPointer = ref this.GetKeyPointerRef(ref key, psfOrdinal); + sb.Append(keyPointer.IsNull ? "null" : keyPointer.Key.ToString()); + } + sb.Append("}"); + return sb.ToString(); + } + } +} diff --git a/cs/src/core/Index/PSF/KeyPointer.cs b/cs/src/core/Index/PSF/KeyPointer.cs new file mode 100644 index 000000000..90214a658 --- /dev/null +++ b/cs/src/core/Index/PSF/KeyPointer.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + internal struct KeyPointer + { + /// + /// The previous address in the hash chain. May be for a different PsfOrdinal than this one due to hash collisions. + /// + internal long PrevAddress; + + /// + /// The ordinal of the current . + /// + internal ushort PsfOrdinal; + + /// + /// Flags regarding the PSF. + /// + internal ushort Flags; + + /// + /// The Key returned by the execution. + /// + internal TPSFKey Key; + + private const ushort NullFlag = 0x0001; + + internal bool IsNull + { + get => (this.Flags & NullFlag) == NullFlag; + set => this.Flags = (ushort)(value ? (this.Flags | NullFlag) : (this.Flags & ~NullFlag)); + } + } +} diff --git a/cs/src/core/Index/PSF/PSFChangeTracker.cs b/cs/src/core/Index/PSF/PSFChangeTracker.cs index 9c244ad8b..d13a88c14 100644 --- a/cs/src/core/Index/PSF/PSFChangeTracker.cs +++ b/cs/src/core/Index/PSF/PSFChangeTracker.cs @@ -17,7 +17,7 @@ public enum UpdateOperation } public unsafe class PSFChangeTracker : IDisposable - where TRecordId : struct + where TRecordId : new() { #region External API public TProviderData BeforeData { get; set; } @@ -59,7 +59,7 @@ internal bool FindGroup(long groupId, out int ordinal) internal ref GroupKeysPair GetGroupRef(int ordinal) => ref groups[ordinal]; - internal ref GroupKeysPair FindFreeGroupRef(long groupId, int keySize, long logAddr = Constants.kInvalidAddress) + internal ref GroupKeysPair FindFreeGroupRef(long groupId, long logAddr = Constants.kInvalidAddress) { if (!this.FindGroup(Constants.kInvalidPsfGroupId, out var ordinal)) { @@ -71,14 +71,10 @@ internal ref GroupKeysPair FindFreeGroupRef(long groupId, int keySize, long logA } ref GroupKeysPair ret = ref this.groups[ordinal]; ret.GroupId = groupId; - ret.KeySize = keySize; ret.LogicalAddress = logAddr; return ref ret; } - public void AssignRcuRecordId(ref PSFValue value) - => value.RecordId = this.AfterRecordId; - public void Dispose() { foreach (var group in this.groups) diff --git a/cs/src/core/Index/PSF/PSFCompositeKey.cs b/cs/src/core/Index/PSF/PSFCompositeKey.cs deleted file mode 100644 index 49b7aca9b..000000000 --- a/cs/src/core/Index/PSF/PSFCompositeKey.cs +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -using System; -using System.Runtime.CompilerServices; - -namespace FASTER.core -{ - /// - /// Wraps the set of TPSFKeys for a record in the secondary FasterKV instance. - /// - /// - public unsafe struct PSFCompositeKey - where TPSFKey : struct - { - // This class is entirely a "reinterpret_cast" implementation; there are no data members. - internal void CopyTo(ref PSFCompositeKey other, int keySize, int psfCount) - { - var thisKeysPointer = (byte*)Unsafe.AsPointer(ref this); - var otherKeysPointer = (byte*)Unsafe.AsPointer(ref other); - var len = keySize * psfCount; - Buffer.MemoryCopy(thisKeysPointer, otherKeysPointer, len, len); - } - - /// - /// Get a reference to the key for the PSF identified by psfOrdinal. - /// - /// The ordinal of the PSF in its parent PSFGroup - /// Size of the PSFKey struct - /// A reference to the key for the PSF identified by psfOrdinal. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref TPSFKey GetKeyRef(int psfOrdinal, int keySize) - => ref Unsafe.AsRef((byte*)Unsafe.AsPointer(ref this) + keySize * psfOrdinal); - - /// - /// Equality comparison - /// - /// The key comparer to use - /// The ordinal of the PSF in its parent PSFGroup - /// Size of the PSFKey struct - /// The key to compare to - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool Equals(IFasterEqualityComparer comparer, int psfOrdinal, int keySize, ref TPSFKey otherKey) - => comparer.Equals(ref GetKeyRef(psfOrdinal, keySize), ref otherKey); - - internal class VarLenLength : IVariableLengthStruct> - { - private readonly int size; - - internal VarLenLength(int keySize, int psfCount) => this.size = keySize * psfCount; - - public int GetAverageLength() => this.size; - - public int GetInitialLength(ref Input _) => this.size; - - public int GetLength(ref PSFCompositeKey _) => this.size; - } - - internal class UnusedKeyComparer : IFasterEqualityComparer> - { - public long GetHashCode64(ref PSFCompositeKey cKey) - => throw new PSFInvalidOperationException("Must use the overload with PSF ordinal"); - - public bool Equals(ref PSFCompositeKey cKey1, ref PSFCompositeKey cKey2) - => throw new PSFInvalidOperationException("Must use the overload with PSF ordinal"); - } - internal unsafe struct PtrWrapper : IDisposable - { - private readonly SectorAlignedMemory mem; - private readonly int size; - - internal PtrWrapper(int size, SectorAlignedBufferPool pool) - { - this.size = size; - this.mem = pool.Get(size); - } - - internal void Set(ref TPSFKey key) - => Buffer.MemoryCopy(Unsafe.AsPointer(ref key), mem.GetValidPointer(), this.size, this.size); - - internal ref PSFCompositeKey GetRef() - => ref Unsafe.AsRef>(mem.GetValidPointer()); - - public void Dispose() => mem.Return(); - } - } - - internal interface ICompositeKeyComparer - { - int PSFCount { get; } - - long GetHashCode64(int psfOrdinal, ref TCompositeKey cKey); - - bool Equals(bool isQuery, int psfOrdinal, ref TCompositeKey queryKey, ref TCompositeKey storedKey); - } - - internal class PSFCompositeKeyComparer : ICompositeKeyComparer> - where TPSFKey : struct - { - private readonly IFasterEqualityComparer userComparer; - private readonly int keySize; - public int PSFCount { get; } - - internal PSFCompositeKeyComparer(IFasterEqualityComparer ucmp, int ksize, int psfCount) - { - this.userComparer = ucmp; - this.keySize = ksize; - this.PSFCount = psfCount; - } - - public long GetHashCode64(int psfOrdinal, ref PSFCompositeKey cKey) - => this.userComparer.GetHashCode64(ref cKey.GetKeyRef(psfOrdinal, this.keySize)); - - public bool Equals(bool isQuery, int psfOrdinal, ref PSFCompositeKey queryKey, ref PSFCompositeKey storedKey) - => userComparer.Equals( - // For a query, the composite key consists of only one key, the query key, at ordinal 0. - ref queryKey.GetKeyRef(isQuery ? 0 : psfOrdinal, this.keySize), - ref storedKey.GetKeyRef(psfOrdinal, this.keySize)); - } -} diff --git a/cs/src/core/Index/PSF/PSFFunctions.cs b/cs/src/core/Index/PSF/PSFFunctions.cs index 6bada62a1..4d584f59f 100644 --- a/cs/src/core/Index/PSF/PSFFunctions.cs +++ b/cs/src/core/Index/PSF/PSFFunctions.cs @@ -8,62 +8,57 @@ namespace FASTER.core { /// - /// The Functions for the PSFValue-wrapped Value; mostly pass-through + /// The Functions for the TRecordId (which is the Value param to the secondary FasterKV); mostly pass-through /// /// The type of the result key /// The type of the value - public class PSFFunctions : IFunctions, PSFValue, PSFInputSecondary, + public class PSFFunctions : IFunctions, TRecordId, PSFInputSecondary, PSFOutputSecondary, PSFContext> where TPSFKey: struct where TRecordId: struct { - IPSFValueAccessor> psfValueAccessor; - // TODO: remove stuff that has been moved to PSFOutput.Visit, etc. - internal PSFFunctions(IPSFValueAccessor> accessor) - => this.psfValueAccessor = accessor; - #region Upserts - public bool ConcurrentWriter(ref PSFCompositeKey _, ref PSFValue src, ref PSFValue dst) + public bool ConcurrentWriter(ref CompositeKey _, ref TRecordId src, ref TRecordId dst) { - src.CopyTo(ref dst, this.psfValueAccessor.RecordIdSize, this.psfValueAccessor.PSFCount); + dst = src; return true; } - public void SingleWriter(ref PSFCompositeKey _, ref PSFValue src, ref PSFValue dst) - => src.CopyTo(ref dst, this.psfValueAccessor.RecordIdSize, this.psfValueAccessor.PSFCount); + public void SingleWriter(ref CompositeKey _, ref TRecordId src, ref TRecordId dst) + => dst = src; - public void UpsertCompletionCallback(ref PSFCompositeKey _, ref PSFValue value, PSFContext ctx) + public void UpsertCompletionCallback(ref CompositeKey _, ref TRecordId value, PSFContext ctx) { /* TODO: UpsertCompletionCallback */ } #endregion Upserts #region Reads - public void ConcurrentReader(ref PSFCompositeKey key, ref PSFInputSecondary input, ref PSFValue value, ref PSFOutputSecondary dst) + public void ConcurrentReader(ref CompositeKey key, ref PSFInputSecondary input, ref TRecordId value, ref PSFOutputSecondary dst) => throw new PSFInternalErrorException("PSFOutput.Visit instead of ConcurrentReader should be called on PSF-implementing FasterKVs"); - public unsafe void SingleReader(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue value, ref PSFOutputSecondary dst) + public unsafe void SingleReader(ref CompositeKey _, ref PSFInputSecondary input, ref TRecordId value, ref PSFOutputSecondary dst) => throw new PSFInternalErrorException("PSFOutput.Visit instead of SingleReader should be called on PSF-implementing FasterKVs"); - public void ReadCompletionCallback(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFOutputSecondary output, PSFContext ctx, Status status) + public void ReadCompletionCallback(ref CompositeKey _, ref PSFInputSecondary input, ref PSFOutputSecondary output, PSFContext ctx, Status status) { /* TODO: ReadCompletionCallback */ } #endregion Reads #region RMWs - public void CopyUpdater(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue oldValue, ref PSFValue newValue) + public void CopyUpdater(ref CompositeKey _, ref PSFInputSecondary input, ref TRecordId oldValue, ref TRecordId newValue) => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); - public void InitialUpdater(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue value) + public void InitialUpdater(ref CompositeKey _, ref PSFInputSecondary input, ref TRecordId value) => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); - public bool InPlaceUpdater(ref PSFCompositeKey _, ref PSFInputSecondary input, ref PSFValue value) + public bool InPlaceUpdater(ref CompositeKey _, ref PSFInputSecondary input, ref TRecordId value) => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); - public void RMWCompletionCallback(ref PSFCompositeKey _, ref PSFInputSecondary input, PSFContext ctx, Status status) + public void RMWCompletionCallback(ref CompositeKey _, ref PSFInputSecondary input, PSFContext ctx, Status status) => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); #endregion RMWs - public void DeleteCompletionCallback(ref PSFCompositeKey _, PSFContext ctx) + public void DeleteCompletionCallback(ref CompositeKey _, PSFContext ctx) { /* TODO: DeleteCompletionCallback */ } public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index 07911cef2..8b272ce3a 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Runtime.CompilerServices; @@ -23,7 +22,7 @@ public class PSFGroup : IExecutePSF, PSFValue, PSFInputSecondary, + internal FasterKV, TRecordId, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions> fht; private readonly PSFFunctions functions; internal IPSFDefinition[] psfDefinitions; @@ -35,20 +34,20 @@ public class PSFGroup : IExecutePSF)); private readonly int recordIdSize = (Utility.GetSize(default(TRecordId)) + sizeof(long) - 1) & ~(sizeof(long) - 1); - internal ConcurrentStack, - PSFValue, PSFInputSecondary, + internal ConcurrentStack, + TRecordId, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions>> freeSessions - = new ConcurrentStack, - PSFValue, PSFInputSecondary, + = new ConcurrentStack, + TRecordId, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions>>(); - internal ConcurrentBag, - PSFValue, PSFInputSecondary, + internal ConcurrentBag, + TRecordId, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions>> allSessions - = new ConcurrentBag, - PSFValue, PSFInputSecondary, + = new ConcurrentBag, + TRecordId, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions>>(); /// @@ -59,8 +58,7 @@ PSFOutputSecondary, PSFContext, PSFFunctions this.PSFs.Length; private readonly IFasterEqualityComparer userKeyComparer; - private readonly ICompositeKeyComparer> compositeKeyComparer; - private readonly IPSFValueAccessor> psfValueAccessor; + private readonly KeyAccessor keyAccessor; private readonly SectorAlignedBufferPool bufferPool; @@ -73,7 +71,7 @@ PSFOutputSecondary, PSFContext, PSFFunctions this.Equals(obj as PSFGroup); /// - public bool Equals(PSFGroup other) => other is null ? false : this.Id == other.Id; + public bool Equals(PSFGroup other) => !(other is null) && this.Id == other.Id; /// /// Constructor @@ -90,30 +88,26 @@ public PSFGroup(long id, IPSFDefinition[] defs, PSFRegis this.userKeyComparer = GetUserKeyComparer(); this.PSFs = defs.Select((def, ord) => new PSF(this.Id, ord, def.Name, this)).ToArray(); - this.compositeKeyComparer = new PSFCompositeKeyComparer(this.userKeyComparer, this.keySize, this.PSFCount); - - this.psfValueAccessor = new PSFValue.Accessor(this.PSFCount, this.recordIdSize); + this.keyAccessor = new KeyAccessor(this.userKeyComparer, this.PSFCount, this.keyPointerSize); this.checkpointSettings = regSettings?.CheckpointSettings; - this.functions = new PSFFunctions(psfValueAccessor); - this.fht = new FasterKV, PSFValue, PSFInputSecondary, + this.functions = new PSFFunctions(); + this.fht = new FasterKV, TRecordId, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions>( - regSettings.HashTableSize, new PSFFunctions(psfValueAccessor), + regSettings.HashTableSize, new PSFFunctions(), regSettings.LogSettings, this.checkpointSettings, null /*SerializerSettings*/, - new PSFCompositeKey.UnusedKeyComparer(), - new VariableLengthStructSettings, PSFValue> + new CompositeKey.UnusedKeyComparer(), + new VariableLengthStructSettings, TRecordId> { - keyLength = new PSFCompositeKey.VarLenLength(this.keySize, this.PSFCount), - valueLength = new PSFValue.VarLenLength(this.recordIdSize, this.PSFCount) + keyLength = new CompositeKey.VarLenLength(this.keyPointerSize, this.PSFCount) } - ) - { psfValueAccessor = psfValueAccessor }; + ) { psfKeyAccessor = this.keyAccessor}; this.bufferPool = this.fht.hlog.bufferPool; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private ClientSession, PSFValue, PSFInputSecondary, + private ClientSession, TRecordId, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions> GetSession() { @@ -126,7 +120,7 @@ PSFFunctions> GetSession() } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ReleaseSession(ClientSession, PSFValue, PSFInputSecondary, + private void ReleaseSession(ClientSession, TRecordId, PSFInputSecondary, PSFOutputSecondary, PSFContext, PSFFunctions> session) => this.freeSessions.Push(session); @@ -167,14 +161,14 @@ internal unsafe void MarkChanges(ref GroupKeysPair keysPair) { ref GroupKeys before = ref keysPair.Before; ref GroupKeys after = ref keysPair.After; - ref PSFCompositeKey beforeCompKey = ref before.GetCompositeKeyRef>(); - ref PSFCompositeKey afterCompKey = ref after.GetCompositeKeyRef>(); + ref CompositeKey beforeCompKey = ref before.GetCompositeKeyRef>(); + ref CompositeKey afterCompKey = ref after.GetCompositeKeyRef>(); for (var ii = 0; ii < this.PSFCount; ++ii) { var beforeIsNull = before.IsNullAt(ii); var afterIsNull = after.IsNullAt(ii); var keysEqual = !beforeIsNull && !afterIsNull - && beforeCompKey.GetKeyRef(ii, this.keySize).Equals(afterCompKey.GetKeyRef(ii, this.keySize)); + && beforeCompKey.GetKeyRef(ii, this.keyPointerSize).Equals(afterCompKey.GetKeyRef(ii, this.keyPointerSize)); // IsNull is already set in PSFGroup.ExecuteAndStore. if (!before.IsNullAt(ii) && (after.IsNullAt(ii) || !keysEqual)) @@ -190,25 +184,26 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor { // Note: stackalloc is safe because PendingContext or PSFChangeTracker will copy it to the bufferPool // if needed. On the Insert fast path, we don't want any allocations otherwise; changeTracker is null. - // TODO: Cutoff (psfCount * keySize) per group to use bufferPool to ensure stackalloc doesn't overflow - var keyMemLen = ((keySize * this.PSFCount + sizeof(int) - 1) & ~(sizeof(int) - 1)); - var keyMemInt = stackalloc int[keyMemLen / sizeof(int)]; - for (var ii = 0; ii < keyMemLen / sizeof(int); ++ii) - keyMemInt[ii] = 0; - var keyMem = (byte*)keyMemInt; - ref PSFCompositeKey compositeKey = ref Unsafe.AsRef>(keyMem); + // TODO: Cutoff (psfCount * keyPointerSize) per group to use bufferPool so stackalloc doesn't overflow. + var keyMemLen = this.keyPointerSize * this.PSFCount; + var keyBytes = stackalloc byte[keyMemLen]; var flagsMemLen = this.PSFCount * sizeof(PSFResultFlags); PSFResultFlags* flags = stackalloc PSFResultFlags[this.PSFCount]; var anyMatch = false; - for (int ii = 0; ii < this.psfDefinitions.Length; ++ii) + + for (var ii = 0; ii < this.PSFCount; ++ii) { + ref KeyPointer keyPointer = ref Unsafe.AsRef>(keyBytes + ii * this.keyPointerSize); + keyPointer.PrevAddress = Constants.kInvalidAddress; + keyPointer.PsfOrdinal = (ushort)ii; // TODO set limit in # of psfs to 2 << 15 + var key = this.psfDefinitions[ii].Execute(providerData); + keyPointer.IsNull = !key.HasValue; *(flags + ii) = key.HasValue ? PSFResultFlags.None : PSFResultFlags.IsNull; if (key.HasValue) { - ref TPSFKey keyPtr = ref Unsafe.AsRef(keyMem + keySize * ii); - keyPtr = key.Value; + keyPointer.Key = key.Value; anyMatch = true; } } @@ -216,21 +211,19 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor if (!anyMatch && phase == PSFExecutePhase.Insert) return Status.OK; - var input = new PSFInputSecondary(0, compositeKeyComparer, this.Id, flags); - - var valueMem = stackalloc byte[recordIdSize + sizeof(long) * this.PSFCount]; - ref PSFValue psfValue = ref Unsafe.AsRef>(valueMem); - psfValue.RecordId = recordId; + ref CompositeKey compositeKey = ref Unsafe.AsRef>(keyBytes); + var input = new PSFInputSecondary(0, this.keyAccessor, this.Id, flags); + var value = recordId; int groupOrdinal = -1; if (!(changeTracker is null)) { - psfValue.RecordId = changeTracker.BeforeRecordId; + value = changeTracker.BeforeRecordId; if (phase == PSFExecutePhase.PreUpdate) { // Get a free group ref and store the "before" values. - ref GroupKeysPair groupKeysPair = ref changeTracker.FindFreeGroupRef(this.Id, this.keySize); - StoreKeys(ref groupKeysPair.Before, keyMem, keyMemLen, flags, flagsMemLen); + ref GroupKeysPair groupKeysPair = ref changeTracker.FindFreeGroupRef(this.Id); + StoreKeys(ref groupKeysPair.Before, keyBytes, keyMemLen, flags, flagsMemLen); return Status.OK; } @@ -244,7 +237,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor else { ref GroupKeysPair groupKeysPair = ref changeTracker.GetGroupRef(groupOrdinal); - StoreKeys(ref groupKeysPair.After, keyMem, keyMemLen, flags, flagsMemLen); + StoreKeys(ref groupKeysPair.After, keyBytes, keyMemLen, flags, flagsMemLen); this.MarkChanges(ref groupKeysPair); // TODOtest: In debug, for initial dev, follow chains to assert the values match what is in the record's compositeKey if (!groupKeysPair.HasChanges) @@ -255,20 +248,16 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor // We don't need to do anything here for Delete. } - long* chainLinkPtrs = this.fht.psfValueAccessor.GetChainLinkPtrs(ref psfValue); - for (int ii = 0; ii < this.psfDefinitions.Length; ++ii) - *(chainLinkPtrs + ii) = Constants.kInvalidAddress; - var session = this.GetSession(); try { var lsn = session.ctx.serialNum + 1; return phase switch { - PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey, ref psfValue, ref input, lsn), + PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey, ref value, ref input, lsn), PSFExecutePhase.PostUpdate => session.PsfUpdate(ref changeTracker.GetGroupRef(groupOrdinal), - ref psfValue, ref input, lsn, changeTracker), - PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey, ref psfValue, ref input, lsn, + ref value, ref input, lsn, changeTracker), + PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey, ref value, ref input, lsn, changeTracker), _ => throw new PSFInternalErrorException("Unknown PSF execution Phase {phase}") }; @@ -330,29 +319,24 @@ public Status Delete(PSFChangeTracker changeTracker) /// public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key) { - // Create a varlen CompositeKey with just one item. This is ONLY used as the query key to - // PSFCompositeKeyComparer. Iterator functions cannot contain unsafe code or have byref args, - // and bufferPool is needed here because the stack goes away as part of the iterator operation. - var keyPtr = new PSFCompositeKey.PtrWrapper(this.keySize, this.bufferPool); - keyPtr.Set(ref key); - - // These indirections are necessary to avoid issues with passing ref or unsafe to an enumerator function. + // Putting the query key in PSFInput is necessary because iterator functions cannot contain unsafe code or have + // byref args, and bufferPool is needed here because the stack goes away as part of the iterator operation. // TODOperf: PSFInput* and PSFOutput* are classes because we communicate through interfaces to avoid // having to add additional generic args. Interfaces on structs incur boxing overhead (plus the complexity // of mutable structs). But check the performance here; if necessary perhaps I can figure out a way to // pass a struct with no TPSFKey, TRecordId, etc. and use an FHT-level interface to manage it. - var psfInput = new PSFInputSecondary(psfOrdinal, compositeKeyComparer, this.Id); - return Query(keyPtr, psfInput); + var psfInput = new PSFInputSecondary(psfOrdinal, this.keyAccessor, this.Id); + psfInput.SetQueryKey(this.bufferPool, ref key); + return Query(psfInput); } - private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKeyPtr, - PSFInputSecondary input) + private IEnumerable Query(PSFInputSecondary input) { // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr. Also consider // having TPSFKeys[] for a single PSF walk through in parallel, so the FHT log memory access is sequential. - var secondaryOutput = new PSFOutputSecondary(this.psfValueAccessor); - var readArgs = new PSFReadArgs, PSFValue>(input, secondaryOutput); + var secondaryOutput = new PSFOutputSecondary(this.keyAccessor); + var readArgs = new PSFReadArgs, TRecordId>(input, secondaryOutput); var session = this.GetSession(); HashSet deadRecs = null; @@ -360,7 +344,7 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe { // Because we traverse the chain, we must wait for any pending read operations to complete. // TODOperf: See if there is a better solution than spinWaiting in CompletePending. - Status status = session.PsfReadKey(ref queryKeyPtr.GetRef(), ref readArgs, session.ctx.serialNum + 1); + Status status = session.PsfReadKey(ref input.QueryKeyRef, ref readArgs, session.ctx.serialNum + 1); if (status == Status.PENDING) session.CompletePending(spinWait: true); if (status != Status.OK) // TODOerr: check other status @@ -404,7 +388,7 @@ private IEnumerable Query(PSFCompositeKey.PtrWrapper queryKe finally { this.ReleaseSession(session); - queryKeyPtr.Dispose(); + input.Dispose(); } } } diff --git a/cs/src/core/Index/PSF/PSFInput.cs b/cs/src/core/Index/PSF/PSFInput.cs index 046969602..d9956d120 100644 --- a/cs/src/core/Index/PSF/PSFInput.cs +++ b/cs/src/core/Index/PSF/PSFInput.cs @@ -11,7 +11,7 @@ namespace FASTER.core /// /// The additional input interface passed to the PSF functions for internal Insert, Read, etc. operations. /// - /// The type of the Key, either a for the + /// The type of the Key, either a for the /// secondary FasterKV instances, or the user's TKVKey for the primary FasterKV instance. /// The interface separation is needed for the PendingContext, and for the "TPSFKey : struct" /// constraint in PSFInputSecondary @@ -45,21 +45,7 @@ public interface IPSFInput /// long ReadLogicalAddress { get; set; } - /// - /// Get the hashcode of the key of the at - /// - long GetHashCode64At(ref TKey cKey); - - /// - /// Determine if the keys match for the s - /// at . For query, the queryKey is a composite consisting of only one key, and - /// its ordinal is always zero. - /// - /// The composite key whose value is being matched with the store. If the - /// operation is Query, this is a composite consisting of only one key, the query key - /// The composite key in storage being compared to - /// - bool EqualsAt(ref TKey queryKey, ref TKey storedKey); + ref TKey QueryKeyRef { get; } } // TODO: Trim IPSFInput to only what PSFInputPrimaryReadAddress needs @@ -99,33 +85,39 @@ public bool IsDelete public long ReadLogicalAddress { get; set; } - public long GetHashCode64At(ref TKey key) - => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); - - public bool EqualsAt(ref TKey queryKey, ref TKey storedKey) - => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); + public ref TKey QueryKeyRef => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); } /// /// Input to operations on the secondary FasterKV instance (stores PSF chains) for everything /// except reading based on a LogicalAddress. /// - public unsafe class PSFInputSecondary : IPSFInput> + public unsafe class PSFInputSecondary : IPSFInput>, IDisposable where TPSFKey : struct { - internal readonly ICompositeKeyComparer> comparer; + internal readonly KeyAccessor keyAccessor; internal PSFResultFlags* resultFlags; + private SectorAlignedMemory keyPointerMem; - internal PSFInputSecondary(int psfOrd, ICompositeKeyComparer> keyCmp, - long groupId, PSFResultFlags* flags = null) + internal PSFInputSecondary(int psfOrdinal, KeyAccessor keyAcc, long groupId, PSFResultFlags* flags = null) { - this.PsfOrdinal = psfOrd; - this.comparer = keyCmp; + this.PsfOrdinal = psfOrdinal; + this.keyAccessor = keyAcc; this.GroupId = groupId; this.resultFlags = flags; this.ReadLogicalAddress = Constants.kInvalidAddress; } + internal void SetQueryKey(SectorAlignedBufferPool pool, ref TPSFKey key) + { + // Create a varlen CompositeKey with just one item. This is ONLY used as the query key to QueryPSF. + this.keyPointerMem = pool.Get(this.keyAccessor.KeyPointerSize); + ref KeyPointer keyPointer = ref Unsafe.AsRef>(keyPointerMem.GetValidPointer()); + keyPointer.PrevAddress = Constants.kInvalidAddress; + keyPointer.PsfOrdinal = (ushort)this.PsfOrdinal; + keyPointer.Key = key; + } + public long GroupId { get; } public int PsfOrdinal { get; set; } @@ -138,14 +130,13 @@ internal PSFInputSecondary(int psfOrd, ICompositeKeyComparer this.resultFlags is null; + public ref CompositeKey QueryKeyRef + => ref Unsafe.AsRef>(this.keyPointerMem.GetValidPointer()); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetHashCode64At(ref PSFCompositeKey cKey) - => this.comparer.GetHashCode64(this.IsQuery ? 0 : this.PsfOrdinal, ref cKey); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool EqualsAt(ref PSFCompositeKey queryKey, ref PSFCompositeKey storedKey) - => this.comparer.Equals(this.IsQuery, this.PsfOrdinal, ref queryKey, ref storedKey); + public void Dispose() + { + if (!(this.keyPointerMem is null)) + this.keyPointerMem.Return(); + } } } diff --git a/cs/src/core/Index/PSF/PSFOutput.cs b/cs/src/core/Index/PSF/PSFOutput.cs index 98db30d04..afaee45e3 100644 --- a/cs/src/core/Index/PSF/PSFOutput.cs +++ b/cs/src/core/Index/PSF/PSFOutput.cs @@ -4,6 +4,7 @@ #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member using System.Collections.Concurrent; +using System.Diagnostics; using System.Runtime.CompilerServices; namespace FASTER.core @@ -11,13 +12,15 @@ namespace FASTER.core /// /// The additional output interface passed to the PSF functions for internal Insert, Read, etc. operations. /// - /// The type of the Key, either a for the + /// The type of the Key, either a for the /// secondary FasterKV instances, or the user's TKVKey for the primary FasterKV instance. - /// The type of the Key, either a for the + /// The type of the Key, either a TRecordId for the /// secondary FasterKV instances, or the user's TKVValue for the primary FasterKV instance. public interface IPSFOutput { PSFOperationStatus Visit(int psfOrdinal, ref TKey key, ref TValue value, bool tombstone, bool isConcurrent); + + PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, ref TValue value, bool tombstone, bool isConcurrent); } /// @@ -48,6 +51,9 @@ public PSFOperationStatus Visit(int psfOrdinal, ref TKVKey key, ref TKVValue val this.ProviderDatas.Enqueue(new FasterKVProviderData(this.allocator, ref key, ref value)); return new PSFOperationStatus(OperationStatus.SUCCESS); } + + public PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, ref TKVValue value, bool tombstone, bool isConcurrent) + => throw new PSFInternalErrorException("Cannot call this form of Visit() on the primary FKV"); // TODO review error messages } /// @@ -55,11 +61,11 @@ public PSFOperationStatus Visit(int psfOrdinal, ref TKVKey key, ref TKVValue val /// /// The type of the key returned from a /// The type of the provider's record identifier - public unsafe class PSFOutputSecondary : IPSFOutput, PSFValue> + public unsafe class PSFOutputSecondary : IPSFOutput, TRecordId> where TPSFKey : struct where TRecordId : struct { - readonly IPSFValueAccessor> psfValueAccessor; + private KeyAccessor keyAccessor; internal TRecordId RecordId { get; private set; } @@ -68,21 +74,35 @@ public unsafe class PSFOutputSecondary : IPSFOutput> accessor) + internal PSFOutputSecondary(KeyAccessor keyAcc) { - this.psfValueAccessor = accessor; + this.keyAccessor = keyAcc; this.RecordId = default; this.PreviousLogicalAddress = Constants.kInvalidAddress; } + public PSFOperationStatus Visit(int psfOrdinal, ref CompositeKey key, + ref TRecordId value, bool tombstone, bool isConcurrent) + { + // This is the secondary FKV; we hold onto the RecordId and create the provider data when QueryPSF returns. + this.RecordId = value; + this.Tombstone = tombstone; + ref KeyPointer keyPointer = ref this.keyAccessor.GetKeyPointerRef(ref key, psfOrdinal); + Debug.Assert(keyPointer.PsfOrdinal == (ushort)psfOrdinal, "Visit found mismatched PSF ordinal"); + this.PreviousLogicalAddress = keyPointer.PrevAddress; + return new PSFOperationStatus(OperationStatus.SUCCESS); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] - public PSFOperationStatus Visit(int psfOrdinal, ref PSFCompositeKey key, - ref PSFValue value, bool tombstone, bool isConcurrent) + public PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, + ref TRecordId value, bool tombstone, bool isConcurrent) { - // This is the secondary FKV so we wait to create provider data until QueryPSF returns. - this.RecordId = value.RecordId; + // This is the secondary FKV; we hold onto the RecordId and create the provider data when QueryPSF returns. + this.RecordId = value; this.Tombstone = tombstone; - this.PreviousLogicalAddress = *(this.psfValueAccessor.GetChainLinkPtrs(ref value) + psfOrdinal); + ref KeyPointer keyPointer = ref this.keyAccessor.GetKeyPointerRef(physicalAddress); + Debug.Assert(keyPointer.PsfOrdinal == (ushort)psfOrdinal, "Visit found mismatched PSF ordinal"); + this.PreviousLogicalAddress = keyPointer.PrevAddress; return new PSFOperationStatus(OperationStatus.SUCCESS); } } diff --git a/cs/src/core/Index/PSF/PSFValue.cs b/cs/src/core/Index/PSF/PSFValue.cs deleted file mode 100644 index 5db008947..000000000 --- a/cs/src/core/Index/PSF/PSFValue.cs +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -using System; -using System.Runtime.CompilerServices; -using System.Text; - -namespace FASTER.core -{ - /// - /// The TKVValue in the secondary, PSF-implementing FasterKV instances; it wraps the - /// and stores the links in the TPSFKey chains. - /// - /// The type of the provider's record identifier - public unsafe struct PSFValue - where TRecordId : struct - { - /// - /// LogicalAddress for FasterKV and FasterLog; something else for another data provider. - /// - public TRecordId RecordId; - - internal void CopyTo(ref PSFValue other, int recordIdSize, int psfCount) - { - other.RecordId = this.RecordId; - var thisChainPointer = ((byte*)Unsafe.AsPointer(ref this) + recordIdSize); - var otherChainPointer = ((byte*)Unsafe.AsPointer(ref other) + recordIdSize); - var len = sizeof(long) * psfCount; // The chains links are "long logicalAddress". - Buffer.MemoryCopy(thisChainPointer, otherChainPointer, len, len); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal byte* GetRecordIdPtr() - => (byte*)Unsafe.AsPointer(ref this); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal long* GetChainLinkPtrs(int recordIdSize) - => (long*)((byte*)Unsafe.AsPointer(ref this) + recordIdSize); - - /// - public override string ToString() => $"RecordId = {this.RecordId}"; - - internal class VarLenLength : IVariableLengthStruct> - { - private readonly int size; - - internal VarLenLength(int recordIdSize, int psfCount) => this.size = recordIdSize + sizeof(long) * psfCount; - - public int GetAverageLength() => this.size; - - public int GetInitialLength(ref Input _) => this.size; - - public int GetLength(ref PSFValue _) => this.size; - } - - internal class Accessor : IPSFValueAccessor> - { - internal Accessor(int psfCount, int recordIdSize) - { - this.PSFCount = psfCount; - this.RecordIdSize = recordIdSize; - } - - public int PSFCount { get; } - - public int RecordIdSize { get; } - - public long* GetChainLinkPtrs(ref PSFValue value) - => value.GetChainLinkPtrs(this.RecordIdSize); - - public unsafe void SetRecordId(ref PSFValue value, TRecId recordId) - { - var recIdPtr = value.GetRecordIdPtr(); - Buffer.MemoryCopy(Unsafe.AsPointer(ref recordId), recIdPtr, this.RecordIdSize, this.RecordIdSize); - } - } - } -} From 267716c274f10d03bc3a3b7cc38e766b66341be1 Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Tue, 7 Jul 2020 11:42:14 -0700 Subject: [PATCH 10/19] Faster: - Fix RMW calls to PSF insert on NOTFOUND - Add an option to execute Before PSFs immediately (within Upsert/RMW rather than after it); this is needed for Object values where the update of the underlying value loses the original value - Rehash the TPSFKey hashcode with PSF ordinal - Implement RecordIterator and boolean QueryPSF forms with up to 3 separate TPSFKey specifications - Add PackageId and PackageOutputPath to csproj for local NuGet generation FasterPSFSample: - Add RMW to FasterPSFSample initial inserts; make Input generic on TValue - Add boolean QueryPSF examples and verification --- .../FasterPSFSample/BlittableOrders.cs | 20 +- cs/playground/FasterPSFSample/FPSF.cs | 8 +- .../FasterPSFSample/FasterPSFSample.cs | 310 +++++++++++++----- cs/playground/FasterPSFSample/Input.cs | 12 +- cs/playground/FasterPSFSample/ObjectOrders.cs | 24 +- cs/playground/FasterPSFSample/Output.cs | 2 +- cs/src/core/ClientSession/ClientSession.cs | 2 +- cs/src/core/FASTER.core.csproj | 2 + cs/src/core/Index/FASTER/FASTERImpl.cs | 70 ++-- cs/src/core/Index/Interfaces/IFasterKV.cs | 2 +- cs/src/core/Index/PSF/FasterPSFImpl.cs | 4 +- .../Index/PSF/FasterPSFSessionOperations.cs | 177 ++++++++-- cs/src/core/Index/PSF/GroupKeys.cs | 15 +- cs/src/core/Index/PSF/KeyAccessor.cs | 9 +- cs/src/core/Index/PSF/PSF.cs | 3 +- cs/src/core/Index/PSF/PSFChangeTracker.cs | 42 ++- cs/src/core/Index/PSF/PSFGroup.cs | 11 +- cs/src/core/Index/PSF/PSFManager.cs | 270 +++++++++------ cs/src/core/Index/PSF/PSFQuerySettings.cs | 28 +- cs/src/core/Index/PSF/RecordIterator.cs | 212 ++++++++++++ 20 files changed, 915 insertions(+), 308 deletions(-) create mode 100644 cs/src/core/Index/PSF/RecordIterator.cs diff --git a/cs/playground/FasterPSFSample/BlittableOrders.cs b/cs/playground/FasterPSFSample/BlittableOrders.cs index bfc6c4fd5..ffb670b9e 100644 --- a/cs/playground/FasterPSFSample/BlittableOrders.cs +++ b/cs/playground/FasterPSFSample/BlittableOrders.cs @@ -19,16 +19,16 @@ public struct BlittableOrders : IOrders public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {Count}"; - public class Functions : IFunctions, Context> + public class Functions : IFunctions, Output, Context> { #region Read - public void ConcurrentReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) + public void ConcurrentReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) => dst.Value = value; - public void SingleReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) + public void SingleReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) => dst.Value = value; - public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) + public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) { if (((IOrders)output.Value).MemberTuple != key.MemberTuple) throw new Exception("Read mismatch error!"); @@ -50,19 +50,19 @@ public void UpsertCompletionCallback(ref Key key, ref BlittableOrders value, Con #endregion Upsert #region RMW - public void CopyUpdater(ref Key key, ref Input input, ref BlittableOrders oldValue, ref BlittableOrders newValue) + public void CopyUpdater(ref Key key, ref Input input, ref BlittableOrders oldValue, ref BlittableOrders newValue) => throw new NotImplementedException(); - public void InitialUpdater(ref Key key, ref Input input, ref BlittableOrders value) - => throw new NotImplementedException(); + public void InitialUpdater(ref Key key, ref Input input, ref BlittableOrders value) + => value = input.InitialUpdateValue; - public bool InPlaceUpdater(ref Key key, ref Input input, ref BlittableOrders value) + public bool InPlaceUpdater(ref Key key, ref Input input, ref BlittableOrders value) { - value.ColorArgb = FasterPSFSample.IPUColor; + value.ColorArgb = input.IPUColorInt; return true; } - public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) + public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) => throw new NotImplementedException(); #endregion RMW diff --git a/cs/playground/FasterPSFSample/FPSF.cs b/cs/playground/FasterPSFSample/FPSF.cs index 7fa42f3ae..6da572902 100644 --- a/cs/playground/FasterPSFSample/FPSF.cs +++ b/cs/playground/FasterPSFSample/FPSF.cs @@ -8,13 +8,13 @@ namespace FasterPSFSample { - class FPSF + class FPSF where TValue : IOrders, new() where TOutput : new() - where TFunctions : IFunctions>, new() + where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { - internal IFasterKV, TFunctions> fht { get; set; } + internal IFasterKV, TFunctions> fht { get; set; } private LogFiles logFiles; @@ -30,7 +30,7 @@ internal FPSF(bool useObjectValues, bool useMultiGroup, bool useReadCache) { this.logFiles = new LogFiles(useObjectValues, useReadCache, useMultiGroup ? 3 : 1); - this.fht = new FasterKV, TFunctions>( + this.fht = new FasterKV, TFunctions>( 1L << 20, new TFunctions(), this.logFiles.LogSettings, null, // TODO: add checkpoints useObjectValues ? new SerializerSettings { valueSerializer = () => new TSerializer() } : null); diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs index fce0286fd..ffabc43f8 100644 --- a/cs/playground/FasterPSFSample/FasterPSFSample.cs +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -9,7 +9,6 @@ using System.Diagnostics; using System.Drawing; using System.Linq; -using System.Runtime.CompilerServices; using System.Threading; namespace FasterPSFSample @@ -17,8 +16,11 @@ namespace FasterPSFSample public partial class FasterPSFSample { private const int UpsertCount = 100; - private static int blueCount, mediumCount, mediumBlueCount; - private static int bin7Count; + private static int blueCount, mediumCount, bin7Count; + private static int intersectMediumBlueCount, unionMediumBlueCount; + private static int intersectMediumBlue7Count, unionMediumBlue7Count; + private static int unionMediumLargeCount, unionRedBlueCount; + private static int unionMediumLargeRedBlueCount, intersectMediumLargeRedBlueCount; internal static Dictionary keyDict = new Dictionary(); @@ -27,8 +29,6 @@ public partial class FasterPSFSample private static int nextId = 1000000000; - internal static int IPUColor; - internal static int serialNo; static void Main(string[] argv) @@ -37,26 +37,28 @@ static void Main(string[] argv) return; if (useObjectValue) // TODO add VarLenValue - RunSample, ObjectOrders.Functions, ObjectOrders.Serializer>(); + RunSample, Output, ObjectOrders.Functions, ObjectOrders.Serializer>(); else - RunSample, BlittableOrders.Functions, NoSerializer>(); + RunSample, Output, BlittableOrders.Functions, NoSerializer>(); return; } - internal static void RunSample() + internal static void RunSample() where TValue : IOrders, new() + where TInput : IInput, new() where TOutput : IOutput, new() - where TFunctions : IFunctions>, new() + where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { - var fpsf = new FPSF(useObjectValue, useMultiGroups, useReadCache: true); + var fpsf = new FPSF(useObjectValue, useMultiGroups, useReadCache: true); try { CountBinKey.WantLastBin = false; - RunUpserts(fpsf); + RunInitialInserts(fpsf); CountBinKey.WantLastBin = true; RunReads(fpsf); - var ok = QueryPSFs(fpsf) + var ok = QueryPSFsWithoutBoolOps(fpsf) + && QueryPSFsWithBoolOps(fpsf) && UpdateSizeByUpsert(fpsf) && UpdateColorByRMW(fpsf) && UpdateCountByUpsert(fpsf) @@ -79,10 +81,11 @@ internal static void RunSample() [Conditional("PSF_TRACE")] static void PsfTraceLine(string message) => Console.WriteLine(message); - internal static void RunUpserts(FPSF fpsf) + internal static void RunInitialInserts(FPSF fpsf) where TValue : IOrders, new() + where TInput : IInput, new() where TOutput : IOutput, new() - where TFunctions : IFunctions>, new() + where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { Console.WriteLine("Writing keys from 0 to {0} to FASTER", UpsertCount); @@ -90,8 +93,9 @@ internal static void RunUpserts(FPSF(); + var input = default(TInput); - for (int i = 0; i < UpsertCount; i++) + for (int ii = 0; ii < UpsertCount; ii++) { // Leave the last value unassigned from each category (we'll use it to update later) var key = new Key(Interlocked.Increment(ref nextId) - 1); @@ -104,22 +108,62 @@ internal static void RunUpserts(FPSF(FPSF fpsf) + internal static void RunReads(FPSF fpsf) where TValue : IOrders, new() + where TInput : IInput, new() where TOutput : IOutput, new() - where TFunctions : IFunctions>, new() + where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { Console.WriteLine("Reading {0} random keys from FASTER", UpsertCount); @@ -145,7 +190,7 @@ internal static void RunReads(FPSF(); var readCount = UpsertCount * 2; @@ -169,58 +214,158 @@ internal static void RunReads(FPSF(FPSF fpsf) + static bool VerifyProviderDatas(FasterKVProviderData[] providerDatas, string name, int expectedCount) where TValue : IOrders, new() + { + Console.Write($"{indent4}{name}: "); + if (verbose) + { + foreach (var providerData in providerDatas) + { + ref TValue value = ref providerData.GetValue(); + Console.WriteLine(indent4 + value); + } + } + Console.WriteLine(providerDatas.Length == expectedCount + ? $"Passed: expected == actual ({expectedCount})" + : $"Failed: expected ({expectedCount}) != actual ({providerDatas.Length})"); + return expectedCount == providerDatas.Length; + } + + internal static bool QueryPSFsWithoutBoolOps(FPSF fpsf) + where TValue : IOrders, new() + where TInput : IInput, new() where TOutput : IOutput, new() - where TFunctions : IFunctions>, new() + where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { Console.WriteLine(); - Console.WriteLine("Querying PSFs from FASTER", UpsertCount); + Console.WriteLine("Querying PSFs from FASTER with no boolean ops", UpsertCount); using var session = fpsf.fht.NewSession(); FasterKVProviderData[] providerDatas = null; var ok = true; - void verifyProviderDatas(string name, int expectedCount) - { - Console.Write($"{indent4}All {name}: "); - if (verbose) - { - foreach (var providerData in providerDatas) - { - ref TValue value = ref providerData.GetValue(); - Console.WriteLine(indent4 + value); - } - } - ok &= expectedCount == providerDatas.Length; - Console.WriteLine(providerDatas.Length == expectedCount - ? $"Passed: expected == actual ({expectedCount})" - : $"Failed: expected ({expectedCount}) != actual ({providerDatas.Length})"); - - } - - // TODO: Intersect/Union (mediumBlueDatas, etc.) - providerDatas = useMultiGroups ? session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium)).ToArray() : session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium)).ToArray(); - verifyProviderDatas("Medium", mediumCount); + ok &= VerifyProviderDatas(providerDatas, "Medium", mediumCount); providerDatas = useMultiGroups ? session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Blue)).ToArray() : session.QueryPSF(fpsf.CombinedColorPsf, new CombinedKey(Color.Blue)).ToArray(); - verifyProviderDatas("Blue", blueCount); + ok &= VerifyProviderDatas(providerDatas, "Blue", blueCount); providerDatas = useMultiGroups ? session.QueryPSF(fpsf.CountBinPsf, new CountBinKey(7)).ToArray() : session.QueryPSF(fpsf.CombinedCountBinPsf, new CombinedKey(7)).ToArray(); - verifyProviderDatas("Bin7", bin7Count); + ok &= VerifyProviderDatas(providerDatas, "Bin7", bin7Count); providerDatas = useMultiGroups ? session.QueryPSF(fpsf.CountBinPsf, new CountBinKey(CountBinKey.LastBin)).ToArray() : session.QueryPSF(fpsf.CombinedCountBinPsf, new CombinedKey(CountBinKey.LastBin)).ToArray(); - verifyProviderDatas("LastBin", 0); // Insert skipped (returned null from the PSF) all that fall into the last bin + ok &= VerifyProviderDatas(providerDatas, "LastBin", 0); // Insert skipped (returned null from the PSF) all that fall into the last bin + return ok; + } + + internal static bool QueryPSFsWithBoolOps(FPSF fpsf) + where TValue : IOrders, new() + where TInput : IInput, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Querying PSFs from FASTER with boolean ops", UpsertCount); + + using var session = fpsf.fht.NewSession(); + FasterKVProviderData[] providerDatas = null; + var ok = true; + + if (useMultiGroups) + { + providerDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), (sz, cl) => sz && cl).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlueCount), intersectMediumBlueCount); + providerDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), (sz, cl) => sz || cl).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlueCount), unionMediumBlueCount); + + providerDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), + fpsf.CountBinPsf, new CountBinKey(7), (sz, cl, ct) => sz && cl && ct).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlue7Count), intersectMediumBlue7Count); + providerDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), + fpsf.CountBinPsf, new CountBinKey(7), (sz, cl, ct) => sz || cl || ct).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlue7Count), unionMediumBlue7Count); + + providerDatas = session.QueryPSF(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeCount), unionMediumLargeCount); + providerDatas = session.QueryPSF(fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionRedBlueCount), unionRedBlueCount); + + providerDatas = session.QueryPSF(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }, + fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }, (sz, cl) => sz && cl).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumLargeRedBlueCount), intersectMediumLargeRedBlueCount); + providerDatas = session.QueryPSF(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }, + fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }, (sz, cl) => sz || cl).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeRedBlueCount), unionMediumLargeRedBlueCount); + } + else + { + providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), (sz, cl) => sz && cl).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlueCount), intersectMediumBlueCount); + providerDatas = session.QueryPSF(new[] { (fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue) }.AsEnumerable()) }, sz => sz[0] && sz[1]).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlueCount), intersectMediumBlueCount); + // --- + providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), (sz, cl) => sz || cl).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlueCount), unionMediumBlueCount); + providerDatas = session.QueryPSF(new[] { (fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue) }.AsEnumerable()) }, sz => sz[0] || sz[1]).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlueCount), unionMediumBlueCount); + // --- + providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), + fpsf.CombinedCountBinPsf, new CombinedKey(7), (sz, cl, ct) => sz && cl && ct).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlue7Count), intersectMediumBlue7Count); + providerDatas = session.QueryPSF(new[] {(fpsf.CombinedSizePsf, new [] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new [] { new CombinedKey(Color.Blue) }.AsEnumerable()), + (fpsf.CombinedCountBinPsf, new [] { new CombinedKey(7) }.AsEnumerable()) }, sz => sz[0] && sz[1] && sz[2]).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlue7Count), intersectMediumBlue7Count); + // --- + providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), + fpsf.CombinedCountBinPsf, new CombinedKey(7), (sz, cl, ct) => sz || cl || ct).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlue7Count), unionMediumBlue7Count); + providerDatas = session.QueryPSF(new[] {(fpsf.CombinedSizePsf, new [] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new [] { new CombinedKey(Color.Blue) }.AsEnumerable()), + (fpsf.CombinedCountBinPsf, new [] { new CombinedKey(7) }.AsEnumerable()) }, sz => sz[0] || sz[1] || sz[2]).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlue7Count), unionMediumBlue7Count); + // --- + providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeCount), unionMediumLargeCount); + providerDatas = session.QueryPSF(fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionRedBlueCount), unionRedBlueCount); + // --- + providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }, + fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }, (sz, cl) => sz && cl).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumLargeRedBlueCount), intersectMediumLargeRedBlueCount); + providerDatas = session.QueryPSF(new[] {(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }.AsEnumerable())}, sz => sz[0] && sz[1]).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumLargeRedBlueCount), intersectMediumLargeRedBlueCount); + // --- + providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }, + fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }, (sz, cl) => sz || cl).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeRedBlueCount), unionMediumLargeRedBlueCount); + providerDatas = session.QueryPSF(new[] {(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }.AsEnumerable())}, sz => sz[0] || sz[1]).ToArray(); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeRedBlueCount), unionMediumLargeRedBlueCount); + } + return ok; } @@ -232,10 +377,11 @@ private static void WriteResult(bool isInitial, string name, int expectedCount, : $"{indent4}{tag} {name} Failed: expected ({expectedCount}) != actual ({actualCount})"); } - internal static bool UpdateSizeByUpsert(FPSF fpsf) + internal static bool UpdateSizeByUpsert(FPSF fpsf) where TValue : IOrders, new() + where TInput : IInput, new() where TOutput : IOutput, new() - where TFunctions : IFunctions>, new() + where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { Console.WriteLine(); @@ -258,13 +404,21 @@ FasterKVProviderData[] GetSizeDatas(Constants.Size size) var context = new Context(); foreach (var providerData in mediumDatas) { - // Update the value - ref TValue value = ref providerData.GetValue(); - Debug.Assert(value.SizeInt == (int)Constants.Size.Medium); - value.SizeInt = (int)Constants.Size.XXLarge; + // Get the old value and confirm it's as expected. + ref TValue oldValue = ref providerData.GetValue(); + Debug.Assert(oldValue.SizeInt == (int)Constants.Size.Medium); + + // Clone the old value with updated Size; note that this cannot modify the "ref providerData.GetValue()" in-place as that will bypass PSFs. + var newValue = new TValue + { + Id = oldValue.Id, + SizeInt = (int)Constants.Size.XXLarge, // updated + ColorArgb = oldValue.ColorArgb, + Count = oldValue.Count + }; // Reuse the same key - session.Upsert(ref providerData.GetKey(), ref value, context, serialNo); + session.Upsert(ref providerData.GetKey(), ref newValue, context, serialNo); RemoveIfSkippedLastBinKey(ref providerData.GetKey()); } @@ -278,10 +432,11 @@ FasterKVProviderData[] GetSizeDatas(Constants.Size size) return ok; } - internal static bool UpdateColorByRMW(FPSF fpsf) + internal static bool UpdateColorByRMW(FPSF fpsf) where TValue : IOrders, new() + where TInput : IInput, new() where TOutput : IOutput, new() - where TFunctions : IFunctions>, new() + where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { Console.WriteLine(); @@ -301,9 +456,8 @@ FasterKVProviderData[] GetColorDatas(Color color) var expected = blueDatas.Length; Console.WriteLine($"{indent2}Changing all Blue to Purple"); - IPUColor = Color.Purple.ToArgb(); var context = new Context(); - var input = default(Input); + var input = new TInput { IPUColorInt = Color.Purple.ToArgb() }; foreach (var providerData in blueDatas) { // This will call Functions<>.InPlaceUpdater. @@ -320,10 +474,11 @@ FasterKVProviderData[] GetColorDatas(Color color) return ok; } - internal static bool UpdateCountByUpsert(FPSF fpsf) + internal static bool UpdateCountByUpsert(FPSF fpsf) where TValue : IOrders, new() + where TInput : IInput, new() where TOutput : IOutput, new() - where TFunctions : IFunctions>, new() + where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { Console.WriteLine(); @@ -351,14 +506,22 @@ FasterKVProviderData[] GetCountDatas(int bin) var context = new Context(); foreach (var providerData in bin7Datas) { - // Update the value - ref TValue value = ref providerData.GetValue(); - Debug.Assert(CountBinKey.GetBin(value.Count, out int tempBin) && tempBin == bin7); - value.Count += (CountBinKey.LastBin - bin7) * CountBinKey.BinSize; - Debug.Assert(CountBinKey.GetBin(value.Count, out tempBin) && tempBin == CountBinKey.LastBin); + // Get the old value and confirm it's as expected. + ref TValue oldValue = ref providerData.GetValue(); + Debug.Assert(CountBinKey.GetBin(oldValue.Count, out int tempBin) && tempBin == bin7); + + // Clone the old value with updated Count; note that this cannot modify the "ref providerData.GetValue()" in-place as that will bypass PSFs. + var newValue = new TValue + { + Id = oldValue.Id, + SizeInt = oldValue.SizeInt, + ColorArgb = oldValue.ColorArgb, + Count = oldValue.Count + (CountBinKey.LastBin - bin7) * CountBinKey.BinSize // updated + }; + Debug.Assert(CountBinKey.GetBin(newValue.Count, out tempBin) && tempBin == CountBinKey.LastBin); // Reuse the same key - session.Upsert(ref providerData.GetKey(), ref value, context, serialNo); + session.Upsert(ref providerData.GetKey(), ref newValue, context, serialNo); } ++serialNo; @@ -373,10 +536,11 @@ FasterKVProviderData[] GetCountDatas(int bin) return ok; } - internal static bool Delete(FPSF fpsf) + internal static bool Delete(FPSF fpsf) where TValue : IOrders, new() + where TInput : IInput, new() where TOutput : IOutput, new() - where TFunctions : IFunctions>, new() + where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { Console.WriteLine(); diff --git a/cs/playground/FasterPSFSample/Input.cs b/cs/playground/FasterPSFSample/Input.cs index 65fb779a5..79b628b6b 100644 --- a/cs/playground/FasterPSFSample/Input.cs +++ b/cs/playground/FasterPSFSample/Input.cs @@ -3,7 +3,17 @@ namespace FasterPSFSample { - public struct Input + public interface IInput { + TValue InitialUpdateValue { get; set; } + + int IPUColorInt { get; set; } + } + + public struct Input : IInput + { + public TValue InitialUpdateValue { get; set; } + + public int IPUColorInt { get; set; } } } diff --git a/cs/playground/FasterPSFSample/ObjectOrders.cs b/cs/playground/FasterPSFSample/ObjectOrders.cs index c0b87301e..072f99ddc 100644 --- a/cs/playground/FasterPSFSample/ObjectOrders.cs +++ b/cs/playground/FasterPSFSample/ObjectOrders.cs @@ -24,8 +24,6 @@ public class ObjectOrders : IOrders public int[] values = new int[numValues]; - public ObjectOrders() => throw new InvalidOperationException("Must use ctor overload"); - public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {Count}"; public class Serializer : BinaryObjectSerializer @@ -44,16 +42,18 @@ public override void Serialize(ref ObjectOrders obj) } } - public class Functions : IFunctions, Context> + public class Functions : IFunctions, Output, Context> { + public ObjectOrders InitialUpdateValue { get; set; } + #region Read - public void ConcurrentReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) + public void ConcurrentReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) => dst.Value = value; - public void SingleReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) + public void SingleReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) => dst.Value = value; - public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) + public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) { if (((IOrders)output.Value).MemberTuple != key.MemberTuple) throw new Exception("Read mismatch error!"); @@ -75,19 +75,19 @@ public void UpsertCompletionCallback(ref Key key, ref ObjectOrders value, Contex #endregion Upsert #region RMW - public void CopyUpdater(ref Key key, ref Input input, ref ObjectOrders oldValue, ref ObjectOrders newValue) + public void CopyUpdater(ref Key key, ref Input input, ref ObjectOrders oldValue, ref ObjectOrders newValue) => throw new NotImplementedException(); - public void InitialUpdater(ref Key key, ref Input input, ref ObjectOrders value) - => throw new NotImplementedException(); + public void InitialUpdater(ref Key key, ref Input input, ref ObjectOrders value) + => value = input.InitialUpdateValue; - public bool InPlaceUpdater(ref Key key, ref Input input, ref ObjectOrders value) + public bool InPlaceUpdater(ref Key key, ref Input input, ref ObjectOrders value) { - value.ColorArgb = FasterPSFSample.IPUColor; + value.ColorArgb = input.IPUColorInt; return true; } - public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) + public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) => throw new NotImplementedException(); #endregion RMW diff --git a/cs/playground/FasterPSFSample/Output.cs b/cs/playground/FasterPSFSample/Output.cs index 3690c0cde..102e79184 100644 --- a/cs/playground/FasterPSFSample/Output.cs +++ b/cs/playground/FasterPSFSample/Output.cs @@ -5,7 +5,7 @@ namespace FasterPSFSample { public interface IOutput { - public TValue Value { get; set; } + TValue Value { get; set; } } public struct Output : IOutput diff --git a/cs/src/core/ClientSession/ClientSession.cs b/cs/src/core/ClientSession/ClientSession.cs index 21a28532a..38157fa5d 100644 --- a/cs/src/core/ClientSession/ClientSession.cs +++ b/cs/src/core/ClientSession/ClientSession.cs @@ -194,7 +194,7 @@ public Status RMW(ref Key key, ref Input input, Context userContext, long serial if (SupportAsync) UnsafeSuspendThread(); } - return status == Status.OK && this.fht.PSFManager.HasPSFs + return (status == Status.OK || status == Status.NOTFOUND) && this.fht.PSFManager.HasPSFs ? this.fht.PSFManager.Update(updateArgs.ChangeTracker) : status; } diff --git a/cs/src/core/FASTER.core.csproj b/cs/src/core/FASTER.core.csproj index 1f5be3989..dc366d43c 100644 --- a/cs/src/core/FASTER.core.csproj +++ b/cs/src/core/FASTER.core.csproj @@ -4,6 +4,8 @@ net461;netstandard2.0;netstandard2.1 AnyCPU;x64 8 + Microsoft.FASTER + $(SolutionDir)\build_output\packages diff --git a/cs/src/core/Index/FASTER/FASTERImpl.cs b/cs/src/core/Index/FASTER/FASTERImpl.cs index 8811ddb04..e3e3a4ade 100644 --- a/cs/src/core/Index/FASTER/FASTERImpl.cs +++ b/cs/src/core/Index/FASTER/FASTERImpl.cs @@ -225,8 +225,7 @@ internal OperationStatus InternalRead( private FasterKVProviderData CreateProviderData(ref Key key, long physicalAddress) => new FasterKVProviderData(this.hlog, ref key, ref hlog.GetValue(physicalAddress)); - private unsafe static void GetAfterRecordId(PSFChangeTracker, long> changeTracker, - ref Value value) + private unsafe static void GetAfterRecordId(PSFChangeTracker, long> changeTracker, ref Value value) { // This indirection is needed because this is the primary FasterKV. Debug.Assert(typeof(Value) == typeof(long)); @@ -234,19 +233,14 @@ private unsafe static void GetAfterRecordId(PSFChangeTracker, long> changeTracker, - ref Key key, long logicalAddress, long physicalAddress) - { - changeTracker.BeforeRecordId = logicalAddress; - changeTracker.BeforeData = CreateProviderData(ref key, physicalAddress); - } + private void SetBeforeData(PSFChangeTracker, long> changeTracker, ref Key key, long logicalAddress, long physicalAddress, bool isIpu) + // If the value has objects, then an in-place RMW to the data in that object will also affect BeforeData, so we must get the PSFs now. // TODOperf this is in session lock + // TODOdoc: If you Read an object value and modify that fetched "ref value" directly, you will break PSFs (the before data is overwritten before we have + // a chance to see it and create the keys). An Upsert must use a separate value. + => this.PSFManager.SetBeforeData(changeTracker, CreateProviderData(ref key, physicalAddress), logicalAddress, isIpu && this.hlog.ValueHasObjects()); - private void SetAfterData(PSFChangeTracker, long> changeTracker, - ref Key key, long logicalAddress, long physicalAddress) - { - changeTracker.AfterRecordId = logicalAddress; - changeTracker.AfterData = CreateProviderData(ref key, physicalAddress); - } + private void SetAfterData(PSFChangeTracker, long> changeTracker, ref Key key, long logicalAddress, long physicalAddress) + => this.PSFManager.SetAfterData(changeTracker, CreateProviderData(ref key, physicalAddress), logicalAddress); #endregion PSF Utilities /// @@ -326,11 +320,11 @@ internal OperationStatus InternalUpsert( out physicalAddress); } - if (logicalAddress >= hlog.ReadOnlyAddress && this.PSFManager.HasPSFs) + if (logicalAddress >= hlog.ReadOnlyAddress && this.PSFManager.HasPSFs && !hlog.GetInfo(physicalAddress).Tombstone) { // Save the PreUpdate values. psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); - SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: true); psfArgs.ChangeTracker.UpdateOp = UpdateOperation.IPU; } } @@ -475,7 +469,7 @@ internal OperationStatus InternalUpsert( { // The old record was valid but not in mutable range (that's handled above), so this is an RCU psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); - SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: false); SetAfterData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); psfArgs.ChangeTracker.UpdateOp = UpdateOperation.RCU; } @@ -645,11 +639,11 @@ internal OperationStatus InternalRMW( out physicalAddress); } - if (logicalAddress >= hlog.ReadOnlyAddress && this.PSFManager.HasPSFs) + if (logicalAddress >= hlog.ReadOnlyAddress && this.PSFManager.HasPSFs && !hlog.GetInfo(physicalAddress).Tombstone) { // Get the PreUpdate values (or the secondary FKV position in the IPUCache). psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); - SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: true); psfArgs.ChangeTracker.UpdateOp = UpdateOperation.IPU; } } @@ -858,14 +852,14 @@ ref hlog.GetValue(physicalAddress), { // Old logicalAddress is invalid or deleted, so this is an Insert only. psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); - SetBeforeData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); + SetBeforeData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress, isIpu: false); psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Insert; } else { // The old record was valid but not in mutable range (that's handled above), so this is an RCU psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); - SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: false); SetAfterData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); psfArgs.ChangeTracker.UpdateOp = UpdateOperation.RCU; } @@ -1121,6 +1115,14 @@ internal OperationStatus InternalDelete( // Apply tombstone bit to the record hlog.GetInfo(physicalAddress).Tombstone = true; + if (this.PSFManager.HasPSFs) + { + // Get the PreUpdate values (or the secondary FKV position in the IPUCache). + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: false); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; + } + if (WriteDefaultOnDelete) { // Write default value @@ -1129,14 +1131,6 @@ internal OperationStatus InternalDelete( functions.ConcurrentWriter(ref hlog.GetKey(physicalAddress), ref v, ref hlog.GetValue(physicalAddress)); } - if (this.PSFManager.HasPSFs) - { - // Get the PreUpdate values (or the secondary FKV position in the IPUCache). - psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); - SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); - psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; - } - status = OperationStatus.SUCCESS; goto LatchRelease; // Release shared latch (if acquired) } @@ -1148,6 +1142,14 @@ internal OperationStatus InternalDelete( { hlog.GetInfo(physicalAddress).Tombstone = true; + if (this.PSFManager.HasPSFs) + { + // Get the PreUpdate values (or the secondary FKV position in the IPUCache). + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: false); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; + } + if (WriteDefaultOnDelete) { // Write default value @@ -1156,14 +1158,6 @@ internal OperationStatus InternalDelete( functions.ConcurrentWriter(ref hlog.GetKey(physicalAddress), ref v, ref hlog.GetValue(physicalAddress)); } - if (this.PSFManager.HasPSFs) - { - // Get the PreUpdate values (or the secondary FKV position in the IPUCache). - psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); - SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); - psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; - } - status = OperationStatus.SUCCESS; goto LatchRelease; // Release shared latch (if acquired) } @@ -1203,7 +1197,7 @@ internal OperationStatus InternalDelete( { // Get the PreUpdate values (or the secondary FKV position in the IPUCache). psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); - SetBeforeData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); + SetBeforeData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress, isIpu: false); psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; } diff --git a/cs/src/core/Index/Interfaces/IFasterKV.cs b/cs/src/core/Index/Interfaces/IFasterKV.cs index b0ba24d4b..c86418ce7 100644 --- a/cs/src/core/Index/Interfaces/IFasterKV.cs +++ b/cs/src/core/Index/Interfaces/IFasterKV.cs @@ -175,7 +175,7 @@ PSF[] RegisterPSF /// The name of the PSF; must be unique across all PSFGroups in this FasterKV instance /// A Func implementing the PSF, it will be wrapped in a delegate /// Optional registration settings for the secondary FasterKV instances, etc. - /// A FasterKV-specific implementation whose TRecordId is long( + /// A FasterKV-specific implementation whose TRecordId is long PSF RegisterPSF( string psfName, Func psfFunc, PSFRegistrationSettings registrationSettings = null) diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index e8c5c2a26..c4d11c394 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -71,7 +71,7 @@ internal OperationStatus PsfInternalReadKey( var psfInput = psfArgs.Input; var psfOutput = psfArgs.Output; - var hash = this.psfKeyAccessor.GetHashCode64(ref queryKey, 0); // the queryKey has only one key + var hash = this.psfKeyAccessor.GetHashCode64(ref queryKey, psfInput.PsfOrdinal, isQuery:true); // the queryKey has only one key var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); if (sessionCtx.phase != Phase.REST) @@ -343,7 +343,7 @@ internal OperationStatus PsfInternalInsert( } ref CASHelper casHelper = ref casHelpers[psfInput.PsfOrdinal]; - casHelper.hash = this.psfKeyAccessor.GetHashCode64(ref compositeKey, psfInput.PsfOrdinal); + casHelper.hash = this.psfKeyAccessor.GetHashCode64(ref compositeKey, psfInput.PsfOrdinal, isQuery:false); var tag = (ushort)((ulong)casHelper.hash >> Constants.kHashTagShift); if (sessionCtx.phase != Phase.REST) diff --git a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs index 74055cce6..c4f0e3bd4 100644 --- a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs @@ -131,16 +131,16 @@ internal IEnumerable> ReturnProviderDatas(IEnum /// /// The type of the key value to return results for /// The Predicate Subset Function object - /// The key value to return results for + /// The key value to return results for /// Optional query settings for EOS, cancellation, etc. /// An enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - PSF psf, TPSFKey psfKey, PSFQuerySettings querySettings = null) + PSF psf, TPSFKey key, PSFQuerySettings querySettings = null) where TPSFKey : struct { // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. - return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, psfKey, querySettings)); + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, key, querySettings)); } /// @@ -148,24 +148,25 @@ public IEnumerable> QueryPSF( /// /// /// foreach (var providerData in fht.QueryPSF(sizePsf, new TestPSFKey[] { Size.Medium, Size.Large })) {...} + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). /// /// The type of the key value to return results for /// The Predicate Subset Function object - /// A vector of key values to return results for; for example, an OR query on + /// A vector of key values to return results for; for example, an OR query on /// a single PSF, or a range query for a PSF that generates keys identifying bins. /// Optional query settings for EOS, cancellation, etc. /// An enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances - public IEnumerable> QueryPSF - (PSF psf, TPSFKey[] psfKeys, PSFQuerySettings querySettings = null) + public IEnumerable> QueryPSF( + PSF psf, IEnumerable keys, PSFQuerySettings querySettings = null) where TPSFKey : struct { // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. - return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, psfKeys, querySettings)); + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, keys, querySettings)); } /// - /// Issue a query on a two s, each with a single key value. + /// Issue a query on two s, each with a single key value. /// /// /// var providerData in fht.QueryPSF(sizePsf, Size.Medium, colorPsf, Color.Red, (l, r) => l || r)) @@ -174,8 +175,8 @@ public IEnumerable> QueryPSF /// The type of the key value for the second /// The first Predicate Subset Function object /// The second Predicate Subset Function object - /// The key value to return results from the first 's stored values - /// The key value to return results from the second 's stored values + /// The key value to return results from the first 's stored values + /// The key value to return results from the second 's stored values /// A predicate that takes as parameters 1) whether a candidate record matches /// the first PSF, 2) whether the record matches the second PSF, and returns a bool indicating whether the /// record should be part of the result set. For example, an AND query would return true iff both input @@ -184,33 +185,32 @@ public IEnumerable> QueryPSF /// An enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - PSF psf1, TPSFKey1 psfKey1, - PSF psf2, TPSFKey2 psfKey2, + PSF psf1, TPSFKey1 key1, + PSF psf2, TPSFKey2 key2, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct where TPSFKey2 : struct { // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. - return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, psfKey1, psf2, psfKey2, - matchPredicate, querySettings)); + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, key1, psf2, key2, matchPredicate, querySettings)); } /// - /// Issue a query on a two s, each with a vector of key values. + /// Issue a query on two s, each with a vector of key values. /// /// /// foreach (var providerData in fht.QueryPSF( - /// sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }, - /// colorPsf, new TestPSFKey[] { Color.Red, Color.Blue}, + /// sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }, + /// colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }, /// (l, r) => l || r)) /// /// The type of the key value for the first /// The type of the key value for the second /// The first Predicate Subset Function object /// The secojnd Predicate Subset Function object - /// The key values to return results from the first 's stored values - /// The key values to return results from the second 's stored values + /// The key values to return results from the first 's stored values + /// The key values to return results from the second 's stored values /// A predicate that takes as parameters 1) whether a candidate record matches /// the first PSF, 2) whether the record matches the second PSF, and returns a bool indicating whether the /// record should be part of the result set. For example, an AND query would return true iff both input @@ -219,16 +219,91 @@ public IEnumerable> QueryPSFAn enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - PSF psf1, TPSFKey1[] psfKeys1, - PSF psf2, TPSFKey2[] psfKeys2, + PSF psf1, IEnumerable keys1, + PSF psf2, IEnumerable keys2, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct where TPSFKey2 : struct { // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. - return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, psfKeys1, psf2, psfKeys2, - matchPredicate, querySettings)); + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, keys1, psf2, keys2, matchPredicate, querySettings)); + } + + /// + /// Issue a query on three s, each with a single key value. + /// + /// + /// var providerData in fht.QueryPSF(sizePsf, Size.Medium, colorPsf, Color.Red, countPsf, 7, (l, m, r) => l || m || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The type of the key value for the third + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The third Predicate Subset Function object + /// The key value to return results from the first 's stored values + /// The key value to return results from the second 's stored values + /// The key value to return results from the third 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, 3) whether the record matches the third PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + PSF psf1, TPSFKey1 key1, + PSF psf2, TPSFKey2 key2, + PSF psf3, TPSFKey3 key3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, key1, psf2, key2, psf3, key3, matchPredicate, querySettings)); + } + + /// + /// Issue a query on three s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }, + /// colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }, + /// countPsf, new [] { new CountKey(7), new CountKey(42) }, + /// (l, m, r) => l || m || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The type of the key value for the third + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The third Predicate Subset Function object + /// The key values to return results from the first 's stored values + /// The key values to return results from the second 's stored values + /// The key values to return results from the third 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, 3) whether the record matches the third PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + PSF psf1, IEnumerable keys1, + PSF psf2, IEnumerable keys2, + PSF psf3, IEnumerable keys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, keys1, psf2, keys2, psf3, keys3, matchPredicate, querySettings)); } /// @@ -240,6 +315,7 @@ public IEnumerable> QueryPSF ll[0])) + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). /// /// The type of the key value for the vector /// A vector of s and associated keys to be queried @@ -252,7 +328,7 @@ public IEnumerable> QueryPSFAn enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - (PSF psf, TPSFKey[] keys)[] psfsAndKeys, + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey : struct @@ -268,10 +344,11 @@ public IEnumerable> QueryPSF( /// foreach (var providerData in fht.QueryPSF( /// new[] { /// (sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), - /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue})}, - /// new[] {(sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue })}, + /// new[] { + /// (countPsf, new [] { new CountKey(7), new CountKey(9) })}, /// (ll, rr) => ll[0] || rr[0])) + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). /// /// The type of the key value for the first vector's s /// The type of the key value for the second vector's s @@ -289,16 +366,56 @@ public IEnumerable> QueryPSF( /// An enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - (PSF psf1, TPSFKey1[] keys1)[] psfsAndKeys1, - (PSF psf2, TPSFKey2[] keys2)[] psfsAndKeys2, + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys2, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct where TPSFKey2 : struct { // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. - return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psfsAndKeys1, psfsAndKeys2, - matchPredicate, querySettings)); + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psfsAndKeys1, psfsAndKeys2, matchPredicate, querySettings)); + } + + /// + /// Issue a query on multiple keys s for three different key types. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { (sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }) }, + /// new[] { (colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }) }, + /// new[] { (countPsf, new [] { new CountKey(4), new CountKey(7) }) }, + /// (ll, mm, rr) => ll[0] || mm[0] || rr[0])) + /// + /// The type of the key value for the first vector's s + /// The type of the key value for the second vector's s + /// The type of the key value for the third vector's s + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A predicate that takes as a parameters three boolean vectors in parallel with + /// each other, and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of all input vectors are true, + /// else false; an OR query would return true if any element of either input vector is true; and more complex + /// logic could be done depending on the specific PSFs. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psfsAndKeys1, psfsAndKeys2, psfsAndKeys3, matchPredicate, querySettings)); } #endregion PSF Query API for primary FasterKV } diff --git a/cs/src/core/Index/PSF/GroupKeys.cs b/cs/src/core/Index/PSF/GroupKeys.cs index 3805ae0fc..408c76962 100644 --- a/cs/src/core/Index/PSF/GroupKeys.cs +++ b/cs/src/core/Index/PSF/GroupKeys.cs @@ -27,13 +27,12 @@ internal unsafe ref TCompositeKey GetCompositeKeyRef() public unsafe bool IsUnlinkOldAt(int ordinal) => (this.ResultFlags + ordinal)->HasFlag(PSFResultFlags.UnlinkOld); public unsafe bool IsLinkNewAt(int ordinal) => (this.ResultFlags + ordinal)->HasFlag(PSFResultFlags.LinkNew); - public unsafe bool HasChanges => this.ResultFlags->HasFlag(PSFResultFlags.UnlinkOld) - || this.ResultFlags->HasFlag(PSFResultFlags.LinkNew); + public unsafe bool HasChanges => this.ResultFlags->HasFlag(PSFResultFlags.UnlinkOld) || this.ResultFlags->HasFlag(PSFResultFlags.LinkNew); public void Dispose() { - this.compositeKeyMem.Return(); - this.flagsMem.Return(); + this.compositeKeyMem?.Return(); + this.flagsMem?.Return(); } } @@ -47,6 +46,14 @@ internal struct GroupKeysPair : IDisposable internal GroupKeys Before; internal GroupKeys After; + internal GroupKeysPair(long id) + { + this.GroupId = id; + this.LogicalAddress = Constants.kInvalidAddress; + this.Before = default; + this.After = default; + } + internal bool HasAddress => this.LogicalAddress != Constants.kInvalidAddress; internal ref TCompositeKey GetBeforeKey() diff --git a/cs/src/core/Index/PSF/KeyAccessor.cs b/cs/src/core/Index/PSF/KeyAccessor.cs index 211670b6b..631471d93 100644 --- a/cs/src/core/Index/PSF/KeyAccessor.cs +++ b/cs/src/core/Index/PSF/KeyAccessor.cs @@ -24,7 +24,7 @@ internal interface IKeyAccessor long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, long psfOrdinal); - long GetHashCode64(ref TCompositeKey key, int psfOrdinal); + long GetHashCode64(ref TCompositeKey key, int psfOrdinal, bool isQuery); bool Equals(ref TCompositeKey queryKey, long physicalAddress); @@ -69,11 +69,10 @@ public long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, long psfO => logicalAddress - psfOrdinal* this.KeyPointerSize - RecordInfo.GetLength(); // TODO: Assumes all PSFs are present [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetHashCode64(ref CompositeKey key, int psfOrdinal) + public long GetHashCode64(ref CompositeKey key, int psfOrdinal, bool isQuery) { - // TODO: Incorporate psfOrdinal into the hashcode; remember that queryKey has only position 0 though. - ref KeyPointer queryKeyPointer = ref key.GetKeyPointerRef(psfOrdinal, this.KeyPointerSize); - return this.userComparer.GetHashCode64(ref queryKeyPointer.Key); + ref KeyPointer queryKeyPointer = ref key.GetKeyPointerRef(isQuery ? 0 : psfOrdinal, this.KeyPointerSize); + return Utility.GetHashCode(this.userComparer.GetHashCode64(ref queryKeyPointer.Key)) ^ Utility.GetHashCode(psfOrdinal + 1); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/cs/src/core/Index/PSF/PSF.cs b/cs/src/core/Index/PSF/PSF.cs index 567304b52..34951a299 100644 --- a/cs/src/core/Index/PSF/PSF.cs +++ b/cs/src/core/Index/PSF/PSF.cs @@ -43,7 +43,6 @@ internal PSF(long groupId, int psfOrdinal, string name, IQueryPSF /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IEnumerable Query(TPSFKey key) - => this.psfGroup.Query(this.PsfOrdinal, key); + internal IEnumerable Query(TPSFKey key) => this.psfGroup.Query(this.PsfOrdinal, key); } } diff --git a/cs/src/core/Index/PSF/PSFChangeTracker.cs b/cs/src/core/Index/PSF/PSFChangeTracker.cs index d13a88c14..aa141bdee 100644 --- a/cs/src/core/Index/PSF/PSFChangeTracker.cs +++ b/cs/src/core/Index/PSF/PSFChangeTracker.cs @@ -2,6 +2,8 @@ // Licensed under the MIT license. using System; +using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member @@ -19,25 +21,36 @@ public enum UpdateOperation public unsafe class PSFChangeTracker : IDisposable where TRecordId : new() { - #region External API - public TProviderData BeforeData { get; set; } - public TRecordId BeforeRecordId { get; set; } + #region Data API + internal TProviderData BeforeData { get; private set; } + internal TRecordId BeforeRecordId { get; private set; } - public TProviderData AfterData { get; set; } - public TRecordId AfterRecordId { get; set; } + internal void SetBeforeData(TProviderData data, TRecordId recordId) + { + this.BeforeData = data; + this.BeforeRecordId = recordId; + } + + internal TProviderData AfterData { get; private set; } + internal TRecordId AfterRecordId { get; private set; } + + internal void SetAfterData(TProviderData data, TRecordId recordId) + { + this.AfterData = data; + this.AfterRecordId = recordId; + } public UpdateOperation UpdateOp { get; set; } - #endregion External API + #endregion Data API private GroupKeysPair[] groups; + internal bool HasBeforeKeys { get; set; } internal long CachedBeforeLA = Constants.kInvalidAddress; - internal void PrepareGroups(int numGroups) + internal PSFChangeTracker(IEnumerable groupIds) { - this.groups = new GroupKeysPair[numGroups]; - for (var ii = 0; ii < numGroups; ++ii) - this.groups[ii].GroupId = Constants.kInvalidPsfGroupId; + this.groups = groupIds.Select(id => new GroupKeysPair(id)).ToArray(); } internal bool FindGroup(long groupId, out int ordinal) @@ -56,14 +69,13 @@ internal bool FindGroup(long groupId, out int ordinal) return false; } - internal ref GroupKeysPair GetGroupRef(int ordinal) - => ref groups[ordinal]; + internal ref GroupKeysPair GetGroupRef(int ordinal) => ref groups[ordinal]; - internal ref GroupKeysPair FindFreeGroupRef(long groupId, long logAddr = Constants.kInvalidAddress) + internal ref GroupKeysPair FindGroupRef(long groupId, long logAddr = Constants.kInvalidAddress) { - if (!this.FindGroup(Constants.kInvalidPsfGroupId, out var ordinal)) + if (!this.FindGroup(groupId, out var ordinal)) { - // A new group was added while we were populating this; should be quite rare. // TODOtest: this case + // A new group was added while we were populating this changeTracker; should be quite rare. // TODOtest: this case var groups = new GroupKeysPair[this.groups.Length + 1]; Array.Copy(this.groups, groups, this.groups.Length); this.groups = groups; diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index 8b272ce3a..65c08cbb7 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -222,7 +222,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor if (phase == PSFExecutePhase.PreUpdate) { // Get a free group ref and store the "before" values. - ref GroupKeysPair groupKeysPair = ref changeTracker.FindFreeGroupRef(this.Id); + ref GroupKeysPair groupKeysPair = ref changeTracker.FindGroupRef(this.Id); StoreKeys(ref groupKeysPair.Before, keyBytes, keyMemLen, flags, flagsMemLen); return Status.OK; } @@ -271,6 +271,9 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor /// public Status GetBeforeKeys(PSFChangeTracker changeTracker) { + if (changeTracker.HasBeforeKeys) + return Status.OK; + // Obtain the "before" values. TODOcache: try to find TRecordId in the IPUCache first. return ExecuteAndStore(changeTracker.BeforeData, default, PSFExecutePhase.PreUpdate, changeTracker); } @@ -280,6 +283,12 @@ public Status GetBeforeKeys(PSFChangeTracker changeTra /// public Status Update(PSFChangeTracker changeTracker) { + if (changeTracker.UpdateOp == UpdateOperation.Insert) + { + // RMW did not find the record so did an insert. Go through Insert logic here. + return this.ExecuteAndStore(changeTracker.BeforeData, changeTracker.BeforeRecordId, PSFExecutePhase.Insert, changeTracker:null); + } + changeTracker.CachedBeforeLA = Constants.kInvalidAddress; // TODOcache: Find BeforeRecordId in IPUCache if (changeTracker.CachedBeforeLA != Constants.kInvalidAddress) { diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index d2bf8783e..6ee0a1f5a 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -1,12 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. +using FASTER.core.Index.PSF; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; +// TODO: Remove PackageId and PackageOutputPath from csproj when this is folded into master + namespace FASTER.core { internal class PSFManager where TRecordId : struct, IComparable @@ -16,10 +19,12 @@ private readonly ConcurrentDictionary psfNames = new ConcurrentDictionary(); + // Default is to let all streams continue to completion. + private static readonly PSFQuerySettings DefaultQuerySettings = new PSFQuerySettings { OnStreamEnded = (unusedPsf, unusedIndex) => true }; + internal bool HasPSFs => this.psfGroups.Count > 0; - internal Status Upsert(TProviderData data, TRecordId recordId, - PSFChangeTracker changeTracker) + internal Status Upsert(TProviderData data, TRecordId recordId, PSFChangeTracker changeTracker) { // TODO: RecordId locking, to ensure consistency of multiple PSFs if the same record is updated // multiple times; possibly a single Array[N] which is locked on TRecordId.GetHashCode % N. @@ -45,7 +50,6 @@ internal Status Upsert(TProviderData data, TRecordId recordId, internal Status Update(PSFChangeTracker changeTracker) { - changeTracker.PrepareGroups(this.psfGroups.Count); foreach (var group in this.psfGroups.Values) { var status = group.Update(changeTracker); @@ -59,7 +63,6 @@ internal Status Update(PSFChangeTracker changeTracker) internal Status Delete(PSFChangeTracker changeTracker) { - changeTracker.PrepareGroups(this.psfGroups.Count); foreach (var group in this.psfGroups.Values) { var status = group.Delete(changeTracker); @@ -74,7 +77,31 @@ internal Status Delete(PSFChangeTracker changeTracker) internal string[][] GetRegisteredPSFs() => throw new NotImplementedException("TODO"); internal PSFChangeTracker CreateChangeTracker() - => new PSFChangeTracker(); + => new PSFChangeTracker(this.psfGroups.Values.Select(group => group.Id)); + + public Status SetBeforeData(PSFChangeTracker changeTracker, TProviderData data, TRecordId recordId, bool executePSFsNow) + { + changeTracker.SetBeforeData(data, recordId); + if (executePSFsNow) + { + foreach (var group in this.psfGroups.Values) + { + var status = group.GetBeforeKeys(changeTracker); + if (status != Status.OK) + { + // TODOerr: handle errors + } + } + changeTracker.HasBeforeKeys = true; + } + return Status.OK; + } + + public Status SetAfterData(PSFChangeTracker changeTracker, TProviderData data, TRecordId recordId) + { + changeTracker.SetAfterData(data, recordId); + return Status.OK; + } private static long NextGroupId = 0; @@ -90,7 +117,7 @@ private void VerifyIsBlittable() throw new PSFArgumentException("The PSF Key type must be blittable."); } - private void VerifyIsOurPSF(PSF psf) + private void VerifyIsOurPSF(PSF psf) // TODO convert to IPSF externally { if (psf is null) throw new PSFArgumentException($"The PSF cannot be null."); @@ -98,6 +125,24 @@ private void VerifyIsOurPSF(PSF psf) throw new PSFArgumentException($"The PSF {psf.Name} is not registered with this FasterKV."); } + private PSF GetImplementingPSF(IPSF ipsf) + { + if (ipsf is null) + throw new PSFArgumentException($"The PSF cannot be null."); + var psf = ipsf as PSF; + if (psf is null || !this.psfNames.TryGetValue(psf.Name, out Guid id) || id != psf.Id) + throw new PSFArgumentException($"The PSF {psf.Name} is not registered with this FasterKV."); + return psf; + } + + private void VerifyIsOurPSF(IEnumerable<(PSF, IEnumerable)> psfsAndKeys) + { + if (psfsAndKeys is null) + throw new PSFArgumentException($"The PSF enumerable cannot be null."); + foreach (var psfAndKeys in psfsAndKeys) + this.VerifyIsOurPSF(psfAndKeys.Item1); + } + internal PSF RegisterPSF(IPSFDefinition def, PSFRegistrationSettings registrationSettings) where TPSFKey : struct @@ -158,138 +203,161 @@ internal PSF[] RegisterPSF(IPSFDefinition QueryPSF(PSF psf, - TPSFKey psfKey, PSFQuerySettings querySettings) + internal IEnumerable QueryPSF(PSF psf, TPSFKey key, PSFQuerySettings querySettings) where TPSFKey : struct { this.VerifyIsOurPSF(psf); - foreach (var recordId in psf.Query(psfKey)) + querySettings ??= DefaultQuerySettings; + foreach (var recordId in psf.Query(key)) { - if (querySettings != null && querySettings.CancellationToken.IsCancellationRequested) - { - if (querySettings.ThrowOnCancellation) - querySettings.CancellationToken.ThrowIfCancellationRequested(); + if (querySettings.IsCanceled) yield break; - } yield return recordId; } } - internal IEnumerable QueryPSF(PSF psf, - TPSFKey[] psfKeys, PSFQuerySettings querySettings) + internal IEnumerable QueryPSF(PSF psf, IEnumerable keys, PSFQuerySettings querySettings) where TPSFKey : struct { this.VerifyIsOurPSF(psf); + querySettings ??= DefaultQuerySettings; - // TODO implement range queries. This will start retrieval of a stream of returned values for the - // chains for all specified keys for the PSF, returning them via an IEnumerable over a PQ - // (PriorityQueue) that is populated from each key's stream. This is how multiple range query bins - // are handled; the semantics are that a Union (via stream merge) of all records for all keys in the - // array is done. Obviously there will be a tradeoff between the granularity of the bins and the - // overhead of the PQ for the streams returned. - return QueryPSF(psf, psfKeys[0], querySettings); + // The recordIds cannot overlap between keys (unless something's gone wrong), so return them all. + // TODOperf: Consider a PQ ordered on secondary FKV LA so we can walk through in parallel (and in memory sequence) in one PsfRead(Key|Address) loop. + foreach (var key in keys) + { + foreach (var recordId in QueryPSF(psf, key, querySettings)) + { + if (querySettings.IsCanceled) + yield break; + yield return recordId; + } + } } internal IEnumerable QueryPSF( - PSF psf1, TPSFKey1 psfKey1, - PSF psf2, TPSFKey2 psfKey2, + PSF psf1, TPSFKey1 key1, + PSF psf2, TPSFKey2 key2, Func matchPredicate, PSFQuerySettings querySettings) + where TPSFKey1 : struct + where TPSFKey2 : struct { - // TODO: full implementation via PQ - using var e1 = psf1.Query(psfKey1).GetEnumerator(); - using var e2 = psf2.Query(psfKey2).GetEnumerator(); + this.VerifyIsOurPSF(psf1); + this.VerifyIsOurPSF(psf2); + querySettings ??= DefaultQuerySettings; - var e1done = !e1.MoveNext(); - var e2done = !e2.MoveNext(); - - var group1 = this.psfGroups[psf1.GroupId]; - var group2 = this.psfGroups[psf2.GroupId]; - - var cancelToken = querySettings is null ? default : querySettings.CancellationToken; - bool cancellationRequested() - { - if (querySettings is null) - return false; - if (querySettings.ThrowOnCancellation) - cancelToken.ThrowIfCancellationRequested(); - return cancelToken.IsCancellationRequested; - } - - while (!e1done && !e2done) - { - if (cancellationRequested()) - yield break; - - // Descending order by recordId. TODO doc: Require IComparable on TRecordId - var cmp = e1.Current.CompareTo(e2.Current); - var predResult = cmp == 0 - ? matchPredicate(true, true) - : cmp > 0 ? matchPredicate(true, false) : matchPredicate(false, true); - if (predResult) - yield return cmp < 0 ? e1.Current : e2.Current; - - // Let the trailing one catch up - if (cmp <= 0) - e1done = !e1.MoveNext(); - if (cmp >= 0) - e2done = !e2.MoveNext(); - } - - // If all streams are done, normal conclusion. - if (e1done && e2done) - yield break; - - // At least one stream is still alive, but not all. See if they registered a callback. - if (!(querySettings is null) && !(querySettings.OnStreamEnded is null)) - { - if (!querySettings.OnStreamEnded(e1done ? (IPSF)psf1 : psf2, e1done ? 0 : 1)) - yield break; - } - - while ((!e1done || !e2done) && !cancellationRequested()) - { - var predResult = matchPredicate(!e1done, !e2done); - if (predResult) - yield return !e1done ? e1.Current : e2.Current; - if (!(!e1done ? e1 : e2).MoveNext()) - yield break; - } + return new QueryRecordIterator(psf1, this.QueryPSF(psf1, key1, querySettings), psf2, this.QueryPSF(psf2, key2, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0]), querySettings).Run(); } internal IEnumerable QueryPSF( - PSF psf1, TPSFKey1[] psfKeys1, - PSF psf2, TPSFKey2[] psfKeys2, + PSF psf1, IEnumerable keys1, + PSF psf2, IEnumerable keys2, Func matchPredicate, PSFQuerySettings querySettings) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psf1); + this.VerifyIsOurPSF(psf2); + querySettings ??= DefaultQuerySettings; + + return new QueryRecordIterator(psf1, this.QueryPSF(psf1, keys1, querySettings), psf2, this.QueryPSF(psf2, keys2, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0]), querySettings).Run(); + } + + public IEnumerable QueryPSF( + PSF psf1, TPSFKey1 key1, + PSF psf2, TPSFKey2 key2, + PSF psf3, TPSFKey3 key3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + this.VerifyIsOurPSF(psf1); + this.VerifyIsOurPSF(psf2); + this.VerifyIsOurPSF(psf3); + querySettings ??= DefaultQuerySettings; + + return new QueryRecordIterator(psf1, this.QueryPSF(psf1, key1, querySettings), psf2, this.QueryPSF(psf2, key2, querySettings), + psf3, this.QueryPSF(psf3, key3, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0], matchIndicators[2][0]), querySettings).Run(); + } + + public IEnumerable QueryPSF( + PSF psf1, IEnumerable keys1, + PSF psf2, IEnumerable keys2, + PSF psf3, IEnumerable keys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct { - // TODO: Similar range-query/PQ implementation (and first-element only execution) as discussed above. - return QueryPSF(psf1, psfKeys1[0], psf2, psfKeys2[0], matchPredicate, querySettings); + this.VerifyIsOurPSF(psf1); + this.VerifyIsOurPSF(psf2); + this.VerifyIsOurPSF(psf3); + querySettings ??= DefaultQuerySettings; + + return new QueryRecordIterator(psf1, this.QueryPSF(psf1, keys1, querySettings), psf2, this.QueryPSF(psf2, keys2, querySettings), + psf3, this.QueryPSF(psf3, keys3, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0], matchIndicators[2][0]), querySettings).Run(); } - // Power user versions. We could add up to 3. Anything more complicated than - // that, they can just post-process with LINQ. + // Power user versions. Anything more complicated than this the caller can post-process with LINQ. internal IEnumerable QueryPSF( - (PSF psf1, TPSFKey[])[] psfsAndKeys, + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys, Func matchPredicate, - PSFQuerySettings querySettings) + PSFQuerySettings querySettings = null) + where TPSFKey : struct { - // TODO: Not implemented. The input argument to the predicate is the matches to each - // element of psfsAndKeys. - return Array.Empty(); + this.VerifyIsOurPSF(psfsAndKeys); + querySettings ??= DefaultQuerySettings; + + return new QueryRecordIterator(new[] { psfsAndKeys.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))) }, + matchIndicators => matchPredicate(matchIndicators[0]), querySettings).Run(); } internal IEnumerable QueryPSF( - (PSF psf1, TPSFKey1[])[] psfsAndKeys1, - (PSF psf2, TPSFKey2[])[] psfsAndKeys2, + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys2, Func matchPredicate, - PSFQuerySettings querySettings) + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psfsAndKeys1); + this.VerifyIsOurPSF(psfsAndKeys2); + querySettings ??= DefaultQuerySettings; + + return new QueryRecordIterator(new[] {psfsAndKeys1.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))), + psfsAndKeys2.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings)))}, + matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1]), querySettings).Run(); + } + + internal IEnumerable QueryPSF( + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct { - // TODO: Not implemented. The first input argument to the predicate is the matches to each - // element of psfsAndKeys1; the second input argument to the predicate is the matches to each - // element of psfsAndKeys2. - return Array.Empty(); + this.VerifyIsOurPSF(psfsAndKeys1); + this.VerifyIsOurPSF(psfsAndKeys2); + this.VerifyIsOurPSF(psfsAndKeys3); + querySettings ??= DefaultQuerySettings; + + return new QueryRecordIterator(new[] {psfsAndKeys1.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))), + psfsAndKeys2.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))), + psfsAndKeys3.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings)))}, + matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1], matchIndicators[2]), querySettings).Run(); } } } diff --git a/cs/src/core/Index/PSF/PSFQuerySettings.cs b/cs/src/core/Index/PSF/PSFQuerySettings.cs index c05e52fda..5fd25575d 100644 --- a/cs/src/core/Index/PSF/PSFQuerySettings.cs +++ b/cs/src/core/Index/PSF/PSFQuerySettings.cs @@ -11,21 +11,35 @@ namespace FASTER.core /// public class PSFQuerySettings { - /// One or more streams has ended. Input PSF whose stream ended, - /// and the index of that PSF in the parameters (left to right, or in the - /// case of PSF arrays as parameters, left top to bottom, then right top - /// to bottom. - /// + /// One or more streams has ended. Inputs are the PSF whose stream ended and the index of that PSF in the parameters, + /// identified by the 0-based ordinal of the TPSFKey type (TPSFKey1, TPSFKey2, or TPSFkey3 in the QueryPSF overloads) and + /// the 0-based ordinal of the PSF within the TPSFKey type. /// true to continue the enumeration, else false - public Func OnStreamEnded; + public Func OnStreamEnded; /// Cancel the enumeration if set. Can be set by another thread, - /// e.g. one presenting results to a UI, or by StreamEnded. + /// e.g. one presenting results to a UI, or by StreamEnded. /// public CancellationToken CancellationToken { get; set; } /// When cancellation is reqested, simply terminate the enumeration /// without throwing a CancellationException. public bool ThrowOnCancellation { get; set; } + + internal bool IsCanceled + { + get + { + if (this.CancellationToken.IsCancellationRequested) + { + if (this.ThrowOnCancellation) + CancellationToken.ThrowIfCancellationRequested(); + return true; + } + return false; + } + } + + internal bool CancelOnEOS(IPSF psf, (int, int) location) => !(this.OnStreamEnded is null) && !this.OnStreamEnded(psf, location); } } diff --git a/cs/src/core/Index/PSF/RecordIterator.cs b/cs/src/core/Index/PSF/RecordIterator.cs new file mode 100644 index 000000000..08b3c489e --- /dev/null +++ b/cs/src/core/Index/PSF/RecordIterator.cs @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace FASTER.core.Index.PSF +{ + /// + /// A single PSF's stream of recordIds + /// + internal class RecordIterator where TRecordId : IComparable + { + private readonly IEnumerator enumerator; + + // Unfortunately we must sort to do the merge. + internal RecordIterator(IEnumerable enumer) => this.enumerator = enumer.OrderBy(rec => rec).GetEnumerator(); + + internal bool Next() + { + if (!this.IsDone) + this.IsDone = !this.enumerator.MoveNext(); + return !this.IsDone; + } + + internal bool IsDone { get; set; } + + internal TRecordId Current => this.enumerator.Current; + + internal void GetIfLower(ref TRecordId currentLowest) + { + if (!this.IsDone && this.enumerator.Current.CompareTo(currentLowest) < 0) + currentLowest = this.enumerator.Current; + } + + internal bool IsMatch(TRecordId recordId) => !this.IsDone && this.enumerator.Current.CompareTo(recordId) == 0; + } + + /// + /// A single TPSFKey type's vector of its PSFs' streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// + internal class KeyTypeRecordIterator where TRecordId : IComparable + { + private struct PsfRecords + { + internal IPSF psf; + internal RecordIterator iterator; + } + + private readonly int keyTypeOrdinal; + private readonly PsfRecords[] psfRecords; + private readonly PSFQuerySettings querySettings; + private int numDone; + + internal KeyTypeRecordIterator(int keyTypeOrd, IPSF psf1, IEnumerable psfRecordEnumerator1, PSFQuerySettings querySettings) + : this(keyTypeOrd, new[] { new PsfRecords { psf = psf1, iterator = new RecordIterator(psfRecordEnumerator1) } }, querySettings) + { } + + internal KeyTypeRecordIterator(int keyTypeOrd, IEnumerable<(IPSF psf, IEnumerable psfRecEnum)> queryResults, PSFQuerySettings querySettings) + : this(keyTypeOrd, queryResults.Select(tup => new PsfRecords { psf = tup.psf, iterator = new RecordIterator(tup.psfRecEnum) }).ToArray(), querySettings) + { } + + private KeyTypeRecordIterator(int keyTypeOrd, PsfRecords[] psfRecs, PSFQuerySettings querySettings) + { + this.keyTypeOrdinal = keyTypeOrd; + this.psfRecords = psfRecs; + this.querySettings = querySettings; + } + + internal int Count => this.psfRecords.Length; + + internal bool IsDone => this.numDone == this.Count; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void GetIfLower(RecordIterator recordIter, ref TRecordId currentLowest, ref bool isFirst) + { + if (recordIter.IsDone) + return; + if (isFirst) + { + currentLowest = recordIter.Current; + isFirst = false; + return; + } + recordIter.GetIfLower(ref currentLowest); + } + + internal bool Initialize(ref TRecordId currentLowest, ref bool isFirst) + { + foreach (var (recordIter, psfIndex) in this.psfRecords.Select((item, index) => (item.iterator, index))) + { + if (!this.Next(recordIter, psfIndex) || querySettings.IsCanceled) + return false; + this.GetIfLower(recordIter, ref currentLowest, ref isFirst); + } + return true; + } + + private bool Next(RecordIterator recordIter, int psfIndex) + { + if (!recordIter.Next()) + { + ++this.numDone; + if (this.querySettings.CancelOnEOS(this.psfRecords[psfIndex].psf, (this.keyTypeOrdinal, psfIndex))) + return false; + } + return true; + } + + internal bool GetNextLowest(TRecordId previousLowest, ref TRecordId currentLowest, ref bool isFirst) + { + foreach (var (recordIter, psfIndex) in this.psfRecords.Select((item, index) => (item.iterator, index))) + { + if (recordIter.IsDone) + continue; + if (querySettings.IsCanceled) + return false; + if (recordIter.IsMatch(previousLowest) && !this.Next(recordIter, psfIndex)) + return false; + this.GetIfLower(recordIter, ref currentLowest, ref isFirst); + } + return true; + } + + internal void MarkMatchIndicators(TRecordId currentLowest, bool[] matchIndicators) + { + foreach (var (recordIter, psfIndex) in this.psfRecords.Select((item, index) => (item.iterator, index))) + matchIndicators[psfIndex] = recordIter.IsMatch(currentLowest); + } + } + + /// + /// The complete query's PSFs' streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// + internal class QueryRecordIterator where TRecordId : IComparable + { + private readonly KeyTypeRecordIterator[] keyTypeRecordIterators; + private readonly bool[][] matchIndicators; + private readonly PSFQuerySettings querySettings; + private readonly Func callerLambda; + + internal QueryRecordIterator(IPSF psf1, IEnumerable keyRecords1, IPSF psf2, IEnumerable keyRecords2, + Func callerLambda, PSFQuerySettings querySettings) + : this(new[] { + new KeyTypeRecordIterator(0, psf1, keyRecords1, querySettings), + new KeyTypeRecordIterator(1, psf2, keyRecords2, querySettings) + }, callerLambda, querySettings) + { } + + internal QueryRecordIterator(IPSF psf1, IEnumerable keyRecords1, IPSF psf2, IEnumerable keyRecords2, + IPSF psf3, IEnumerable keyRecords3, + Func callerLambda, PSFQuerySettings querySettings) + : this(new[] { + new KeyTypeRecordIterator(0, psf1, keyRecords1, querySettings), + new KeyTypeRecordIterator(1, psf2, keyRecords2, querySettings), + new KeyTypeRecordIterator(2, psf3, keyRecords3, querySettings) + }, callerLambda, querySettings) + { } + + internal QueryRecordIterator(IEnumerable keyRecEnums)>> keyTypeQueryResultsEnum, + Func callerLambda, PSFQuerySettings querySettings) + : this(keyTypeQueryResultsEnum.Select((ktqr, index) => new KeyTypeRecordIterator(index, ktqr, querySettings)).ToArray(), + callerLambda, querySettings) + { } + + private QueryRecordIterator(KeyTypeRecordIterator[] ktris, Func callerLambda, PSFQuerySettings querySettings) + { + this.keyTypeRecordIterators = ktris; + this.matchIndicators = this.keyTypeRecordIterators.Select(ktri => new bool[ktri.Count]).ToArray(); + this.callerLambda = callerLambda; + this.querySettings = querySettings; + } + + internal IEnumerable Run() + { + TRecordId current = default; + bool isFirst = true; + foreach (var keyIter in this.keyTypeRecordIterators) + { + if (!keyIter.Initialize(ref current, ref isFirst)) + yield break; + } + + while(true) + { + var allDone = true; + foreach (var (keyIter, keyIndex) in this.keyTypeRecordIterators.Select((iter, index) => (iter, index))) + { + keyIter.MarkMatchIndicators(current, this.matchIndicators[keyIndex]); + allDone &= keyIter.IsDone; + } + + if (allDone || this.querySettings.IsCanceled) + yield break; + + if (this.callerLambda(this.matchIndicators)) + yield return current; + + var prevLowest = current; + isFirst = true; + foreach (var keyIter in this.keyTypeRecordIterators) + { + // TODOperf: consider a PQ here. Given that we have to go through all matchIndicators anyway, at what number of streams would the additional complexity improve speed? + if (!keyIter.GetNextLowest(prevLowest, ref current, ref isFirst)) + yield break; + } + } + } + } +} From aae18549a9a541a9916ccf1b5a98f8c378fd7e0a Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Tue, 7 Jul 2020 12:22:55 -0700 Subject: [PATCH 11/19] - Change external API to from PSF to IPSF - Remove FasterPSFSample launchsettings.json --- .gitignore | 1 + cs/playground/FasterPSFSample/FPSF.cs | 34 +++---- .../FasterPSFSample/FasterPSFSample.cs | 16 ++-- .../Properties/launchSettings.json | 7 -- cs/src/core/Index/Interfaces/IFasterKV.cs | 12 +-- .../core/Index/PSF/FasterPSFRegistration.cs | 12 +-- .../Index/PSF/FasterPSFSessionOperations.cs | 36 +++---- cs/src/core/Index/PSF/PSFManager.cs | 95 ++++++++++--------- 8 files changed, 102 insertions(+), 111 deletions(-) delete mode 100644 cs/playground/FasterPSFSample/Properties/launchSettings.json diff --git a/.gitignore b/.gitignore index 12971f9e1..00584490b 100644 --- a/.gitignore +++ b/.gitignore @@ -193,3 +193,4 @@ packages/ *.lib nativebin/ /cs/benchmark/Properties/launchSettings.json +/cs/playground/FasterPSFSample/Properties/launchSettings.json diff --git a/cs/playground/FasterPSFSample/FPSF.cs b/cs/playground/FasterPSFSample/FPSF.cs index 6da572902..217a63f0f 100644 --- a/cs/playground/FasterPSFSample/FPSF.cs +++ b/cs/playground/FasterPSFSample/FPSF.cs @@ -14,23 +14,19 @@ class FPSF where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { - internal IFasterKV, TFunctions> fht { get; set; } + internal IFasterKV, TFunctions> FasterKV { get; set; } private LogFiles logFiles; - internal PSF SizePsf; - internal PSF ColorPsf; - internal PSF CountBinPsf; - - internal PSF CombinedSizePsf; - internal PSF CombinedColorPsf; - internal PSF CombinedCountBinPsf; + // MultiGroup PSFs -- different key types, one per group. + internal IPSF SizePsf, ColorPsf, CountBinPsf; + internal IPSF CombinedSizePsf, CombinedColorPsf, CombinedCountBinPsf; internal FPSF(bool useObjectValues, bool useMultiGroup, bool useReadCache) { this.logFiles = new LogFiles(useObjectValues, useReadCache, useMultiGroup ? 3 : 1); - this.fht = new FasterKV, TFunctions>( + this.FasterKV = new FasterKV, TFunctions>( 1L << 20, new TFunctions(), this.logFiles.LogSettings, null, // TODO: add checkpoints useObjectValues ? new SerializerSettings { valueSerializer = () => new TSerializer() } : null); @@ -38,20 +34,20 @@ internal FPSF(bool useObjectValues, bool useMultiGroup, bool useReadCache) if (useMultiGroup) { var psfOrdinal = 0; - this.SizePsf = fht.RegisterPSF("sizePsf", (k, v) => new SizeKey((Constants.Size)v.SizeInt), + this.SizePsf = FasterKV.RegisterPSF(nameof(this.SizePsf), (k, v) => new SizeKey((Constants.Size)v.SizeInt), CreatePSFRegistrationSettings(psfOrdinal++)); - this.ColorPsf = fht.RegisterPSF("colorPsf", (k, v) => new ColorKey(Constants.ColorDict[v.ColorArgb]), + this.ColorPsf = FasterKV.RegisterPSF(nameof(this.ColorPsf), (k, v) => new ColorKey(Constants.ColorDict[v.ColorArgb]), CreatePSFRegistrationSettings(psfOrdinal++)); - this.CountBinPsf = fht.RegisterPSF("countBinPsf", (k, v) => CountBinKey.GetBin(v.Count, out int bin) + this.CountBinPsf = FasterKV.RegisterPSF(nameof(this.CountBinPsf), (k, v) => CountBinKey.GetBin(v.Count, out int bin) ? new CountBinKey(bin) : (CountBinKey?)null, CreatePSFRegistrationSettings(psfOrdinal++)); } else { - var psfs = fht.RegisterPSF(new(string, Func)[] { - ("sizePsf", (k, v) => new CombinedKey((Constants.Size)v.SizeInt)), - ("colorPsf", (k, v) => new CombinedKey(Constants.ColorDict[v.ColorArgb])), - ("countBinPsf", (k, v) => CountBinKey.GetBin(v.Count, out int bin) + var psfs = FasterKV.RegisterPSF(new(string, Func)[] { + (nameof(this.SizePsf), (k, v) => new CombinedKey((Constants.Size)v.SizeInt)), + (nameof(this.ColorPsf), (k, v) => new CombinedKey(Constants.ColorDict[v.ColorArgb])), + (nameof(this.CountBinPsf), (k, v) => CountBinKey.GetBin(v.Count, out int bin) ? new CombinedKey(bin) : (CombinedKey?)null) }, CreatePSFRegistrationSettings(0) ); @@ -88,10 +84,10 @@ PSFRegistrationSettings CreatePSFRegistrationSettings(int psfOrdinal internal void Close() { - if (!(this.fht is null)) + if (!(this.FasterKV is null)) { - this.fht.Dispose(); - this.fht = null; + this.FasterKV.Dispose(); + this.FasterKV = null; } if (!(this.logFiles is null)) { diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs index ffabc43f8..00c2252b4 100644 --- a/cs/playground/FasterPSFSample/FasterPSFSample.cs +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -91,7 +91,7 @@ internal static void RunInitialInserts(); var input = default(TInput); @@ -196,7 +196,7 @@ internal static void RunReads( var keys = keyDict.Keys.ToArray(); - using var session = fpsf.fht.NewSession(); + using var session = fpsf.FasterKV.NewSession(); for (int i = 0; i < UpsertCount; i++) { var key = keys[rng.Next(keys.Length)]; @@ -242,7 +242,7 @@ internal static bool QueryPSFsWithoutBoolOps[] providerDatas = null; var ok = true; @@ -278,7 +278,7 @@ internal static bool QueryPSFsWithBoolOps[] providerDatas = null; var ok = true; @@ -387,7 +387,7 @@ internal static bool UpdateSizeByUpsert[] GetSizeDatas(Constants.Size size) => useMultiGroups @@ -442,7 +442,7 @@ internal static bool UpdateColorByRMW[] GetColorDatas(Color color) => useMultiGroups @@ -484,7 +484,7 @@ internal static bool UpdateCountByUpsert[] GetCountDatas(int bin) @@ -546,7 +546,7 @@ internal static bool Delete(FP Console.WriteLine(); Console.WriteLine("Deleting Colors"); - using var session = fpsf.fht.NewSession(); + using var session = fpsf.FasterKV.NewSession(); FasterKVProviderData[] GetColorDatas(Color color) => useMultiGroups diff --git a/cs/playground/FasterPSFSample/Properties/launchSettings.json b/cs/playground/FasterPSFSample/Properties/launchSettings.json deleted file mode 100644 index 99c85823e..000000000 --- a/cs/playground/FasterPSFSample/Properties/launchSettings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "profiles": { - "FasterPSFSample": { - "commandName": "Project" - } - } -} \ No newline at end of file diff --git a/cs/src/core/Index/Interfaces/IFasterKV.cs b/cs/src/core/Index/Interfaces/IFasterKV.cs index c86418ce7..072e7594d 100644 --- a/cs/src/core/Index/Interfaces/IFasterKV.cs +++ b/cs/src/core/Index/Interfaces/IFasterKV.cs @@ -143,7 +143,7 @@ public interface IFasterKV : IDis /// A FasterKV-specific form of a PSF definition /// Optional registration settings for the secondary FasterKV instances, etc. /// A FasterKV-specific PSF implementation whose TRecordId is long( - PSF RegisterPSF( + IPSF RegisterPSF( FasterKVPSFDefinition def, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct; @@ -160,7 +160,7 @@ PSF RegisterPSF( /// An array of FasterKV-specific forms of PSF definitions /// Optional registration settings for the secondary FasterKV instances, etc. /// A FasterKV-specific PSF implementation whose TRecordId is long( - PSF[] RegisterPSF + IPSF[] RegisterPSF (FasterKVPSFDefinition[] defs, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct; @@ -176,7 +176,7 @@ PSF[] RegisterPSF /// A Func implementing the PSF, it will be wrapped in a delegate /// Optional registration settings for the secondary FasterKV instances, etc. /// A FasterKV-specific implementation whose TRecordId is long - PSF RegisterPSF( + IPSF RegisterPSF( string psfName, Func psfFunc, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct; @@ -193,7 +193,7 @@ PSF RegisterPSF( /// unique across all PSFGroups in this FasterKV instance, and the Func will be wrapped in a delegate /// "params" won't allow the optional fromAddress and keyComparer, so an overload is provided /// to specify those - PSF[] RegisterPSF( + IPSF[] RegisterPSF( params (string, Func)[] psfFuncs) where TPSFKey : struct; @@ -213,7 +213,7 @@ PSF[] RegisterPSF( /// Optional registration settings for the secondary FasterKV instances, etc. /// If the registrationSettings parameters are null, then it is simpler to call the "params" overload /// than to create the vector explicitly - PSF[] RegisterPSF( + IPSF[] RegisterPSF( (string, Func)[] psfDefs, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct; @@ -224,7 +224,7 @@ PSF[] RegisterPSF( /// /// An array of string arrays; each outer array corresponds to a /// - string[][] GetRegisteredPSFs(); + string[][] GetRegisteredPSFNames(); #endregion PSF Registration diff --git a/cs/src/core/Index/PSF/FasterPSFRegistration.cs b/cs/src/core/Index/PSF/FasterPSFRegistration.cs index 3892f94b5..1a4fa5085 100644 --- a/cs/src/core/Index/PSF/FasterPSFRegistration.cs +++ b/cs/src/core/Index/PSF/FasterPSFRegistration.cs @@ -37,21 +37,21 @@ private PSFRegistrationSettings CreateDefaultRegistrationSettings - public PSF RegisterPSF( + public IPSF RegisterPSF( FasterKVPSFDefinition def, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct => this.PSFManager.RegisterPSF(def, registrationSettings ?? CreateDefaultRegistrationSettings()); /// - public PSF[] RegisterPSF + public IPSF[] RegisterPSF (FasterKVPSFDefinition[] defs, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct => this.PSFManager.RegisterPSF(defs, registrationSettings ?? CreateDefaultRegistrationSettings()); /// - public PSF RegisterPSF( + public IPSF RegisterPSF( string psfName, Func psfFunc, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct @@ -59,14 +59,14 @@ public PSF RegisterPSF( registrationSettings ?? CreateDefaultRegistrationSettings()); /// - public PSF[] RegisterPSF( + public IPSF[] RegisterPSF( params (string, Func)[] psfFuncs) where TPSFKey : struct => this.PSFManager.RegisterPSF(psfFuncs.Select(e => new FasterKVPSFDefinition(e.Item1, e.Item2)).ToArray(), CreateDefaultRegistrationSettings()); /// - public PSF[] RegisterPSF( + public IPSF[] RegisterPSF( (string, Func)[] psfDefs, PSFRegistrationSettings registrationSettings = null) where TPSFKey : struct @@ -74,7 +74,7 @@ public PSF[] RegisterPSF( CreateDefaultRegistrationSettings()); /// - public string[][] GetRegisteredPSFs() => this.PSFManager.GetRegisteredPSFs(); + public string[][] GetRegisteredPSFNames() => this.PSFManager.GetRegisteredPSFNames(); #endregion PSF Registration API } } diff --git a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs index c4f0e3bd4..449952929 100644 --- a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs @@ -136,7 +136,7 @@ internal IEnumerable> ReturnProviderDatas(IEnum /// An enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - PSF psf, TPSFKey key, PSFQuerySettings querySettings = null) + IPSF psf, TPSFKey key, PSFQuerySettings querySettings = null) where TPSFKey : struct { // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. @@ -158,7 +158,7 @@ public IEnumerable> QueryPSF( /// An enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - PSF psf, IEnumerable keys, PSFQuerySettings querySettings = null) + IPSF psf, IEnumerable keys, PSFQuerySettings querySettings = null) where TPSFKey : struct { // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. @@ -185,8 +185,8 @@ public IEnumerable> QueryPSF( /// An enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - PSF psf1, TPSFKey1 key1, - PSF psf2, TPSFKey2 key2, + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct @@ -219,8 +219,8 @@ public IEnumerable> QueryPSFAn enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - PSF psf1, IEnumerable keys1, - PSF psf2, IEnumerable keys2, + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct @@ -253,9 +253,9 @@ public IEnumerable> QueryPSFAn enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - PSF psf1, TPSFKey1 key1, - PSF psf2, TPSFKey2 key2, - PSF psf3, TPSFKey3 key3, + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + IPSF psf3, TPSFKey3 key3, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct @@ -293,9 +293,9 @@ public IEnumerable> QueryPSFAn enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - PSF psf1, IEnumerable keys1, - PSF psf2, IEnumerable keys2, - PSF psf3, IEnumerable keys3, + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + IPSF psf3, IEnumerable keys3, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct @@ -328,7 +328,7 @@ public IEnumerable> QueryPSFAn enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey : struct @@ -366,8 +366,8 @@ public IEnumerable> QueryPSF( /// An enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys1, - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct @@ -405,9 +405,9 @@ public IEnumerable> QueryPSFAn enumerable of the FasterKV-specific provider data from the primary FasterKV /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances public IEnumerable> QueryPSF( - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys1, - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys2, - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys3, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys3, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index 6ee0a1f5a..3b3dcc397 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -74,7 +74,7 @@ internal Status Delete(PSFChangeTracker changeTracker) return Status.OK; } - internal string[][] GetRegisteredPSFs() => throw new NotImplementedException("TODO"); + internal string[][] GetRegisteredPSFNames() => throw new NotImplementedException("TODO"); internal PSFChangeTracker CreateChangeTracker() => new PSFChangeTracker(this.psfGroups.Values.Select(group => group.Id)); @@ -117,34 +117,35 @@ private void VerifyIsBlittable() throw new PSFArgumentException("The PSF Key type must be blittable."); } - private void VerifyIsOurPSF(PSF psf) // TODO convert to IPSF externally + private PSF GetImplementingPSF(IPSF ipsf) { - if (psf is null) + if (ipsf is null) throw new PSFArgumentException($"The PSF cannot be null."); - if (!this.psfNames.TryGetValue(psf.Name, out Guid id) || id != psf.Id) - throw new PSFArgumentException($"The PSF {psf.Name} is not registered with this FasterKV."); + var psf = ipsf as PSF; + Guid id = default; + if (psf is null || !this.psfNames.TryGetValue(psf.Name, out id) || id != psf.Id) + throw new PSFArgumentException($"The PSF {psf.Name} with Id {(psf is null ? "(unavailable)" : id.ToString())} is not registered with this FasterKV."); + return psf; } - private PSF GetImplementingPSF(IPSF ipsf) + private void VerifyIsOurPSF(IPSF psf) { - if (ipsf is null) + if (psf is null) throw new PSFArgumentException($"The PSF cannot be null."); - var psf = ipsf as PSF; - if (psf is null || !this.psfNames.TryGetValue(psf.Name, out Guid id) || id != psf.Id) + if (!this.psfNames.ContainsKey(psf.Name)) throw new PSFArgumentException($"The PSF {psf.Name} is not registered with this FasterKV."); - return psf; } - private void VerifyIsOurPSF(IEnumerable<(PSF, IEnumerable)> psfsAndKeys) + private void VerifyIsOurPSF(IEnumerable<(IPSF, IEnumerable)> psfsAndKeys) { if (psfsAndKeys is null) throw new PSFArgumentException($"The PSF enumerable cannot be null."); foreach (var psfAndKeys in psfsAndKeys) - this.VerifyIsOurPSF(psfAndKeys.Item1); + this.VerifyIsOurPSF(psfAndKeys.Item1); } - internal PSF RegisterPSF(IPSFDefinition def, - PSFRegistrationSettings registrationSettings) + internal IPSF RegisterPSF(IPSFDefinition def, + PSFRegistrationSettings registrationSettings) where TPSFKey : struct { this.VerifyIsBlittable(); @@ -165,8 +166,8 @@ internal PSF RegisterPSF(IPSFDefinition[] RegisterPSF(IPSFDefinition[] defs, - PSFRegistrationSettings registrationSettings) + internal IPSF[] RegisterPSF(IPSFDefinition[] defs, + PSFRegistrationSettings registrationSettings) where TPSFKey : struct { this.VerifyIsBlittable(); @@ -203,12 +204,12 @@ internal PSF[] RegisterPSF(IPSFDefinition QueryPSF(PSF psf, TPSFKey key, PSFQuerySettings querySettings) + internal IEnumerable QueryPSF(IPSF psf, TPSFKey key, PSFQuerySettings querySettings) where TPSFKey : struct { - this.VerifyIsOurPSF(psf); + var psfImpl = this.GetImplementingPSF(psf); querySettings ??= DefaultQuerySettings; - foreach (var recordId in psf.Query(key)) + foreach (var recordId in psfImpl.Query(key)) { if (querySettings.IsCanceled) yield break; @@ -216,10 +217,10 @@ internal IEnumerable QueryPSF(PSF psf, T } } - internal IEnumerable QueryPSF(PSF psf, IEnumerable keys, PSFQuerySettings querySettings) + internal IEnumerable QueryPSF(IPSF psf, IEnumerable keys, PSFQuerySettings querySettings) where TPSFKey : struct { - this.VerifyIsOurPSF(psf); + this.VerifyIsOurPSF(psf); querySettings ??= DefaultQuerySettings; // The recordIds cannot overlap between keys (unless something's gone wrong), so return them all. @@ -236,15 +237,15 @@ internal IEnumerable QueryPSF(PSF psf, I } internal IEnumerable QueryPSF( - PSF psf1, TPSFKey1 key1, - PSF psf2, TPSFKey2 key2, + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, Func matchPredicate, PSFQuerySettings querySettings) where TPSFKey1 : struct where TPSFKey2 : struct { - this.VerifyIsOurPSF(psf1); - this.VerifyIsOurPSF(psf2); + this.VerifyIsOurPSF(psf1); + this.VerifyIsOurPSF(psf2); querySettings ??= DefaultQuerySettings; return new QueryRecordIterator(psf1, this.QueryPSF(psf1, key1, querySettings), psf2, this.QueryPSF(psf2, key2, querySettings), @@ -252,15 +253,15 @@ internal IEnumerable QueryPSF( } internal IEnumerable QueryPSF( - PSF psf1, IEnumerable keys1, - PSF psf2, IEnumerable keys2, + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, Func matchPredicate, PSFQuerySettings querySettings) where TPSFKey1 : struct where TPSFKey2 : struct { - this.VerifyIsOurPSF(psf1); - this.VerifyIsOurPSF(psf2); + this.VerifyIsOurPSF(psf1); + this.VerifyIsOurPSF(psf2); querySettings ??= DefaultQuerySettings; return new QueryRecordIterator(psf1, this.QueryPSF(psf1, keys1, querySettings), psf2, this.QueryPSF(psf2, keys2, querySettings), @@ -268,18 +269,18 @@ internal IEnumerable QueryPSF( } public IEnumerable QueryPSF( - PSF psf1, TPSFKey1 key1, - PSF psf2, TPSFKey2 key2, - PSF psf3, TPSFKey3 key3, + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + IPSF psf3, TPSFKey3 key3, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct where TPSFKey2 : struct where TPSFKey3 : struct { - this.VerifyIsOurPSF(psf1); - this.VerifyIsOurPSF(psf2); - this.VerifyIsOurPSF(psf3); + this.VerifyIsOurPSF(psf1); + this.VerifyIsOurPSF(psf2); + this.VerifyIsOurPSF(psf3); querySettings ??= DefaultQuerySettings; return new QueryRecordIterator(psf1, this.QueryPSF(psf1, key1, querySettings), psf2, this.QueryPSF(psf2, key2, querySettings), @@ -288,18 +289,18 @@ public IEnumerable QueryPSF( } public IEnumerable QueryPSF( - PSF psf1, IEnumerable keys1, - PSF psf2, IEnumerable keys2, - PSF psf3, IEnumerable keys3, + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + IPSF psf3, IEnumerable keys3, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct where TPSFKey2 : struct where TPSFKey3 : struct { - this.VerifyIsOurPSF(psf1); - this.VerifyIsOurPSF(psf2); - this.VerifyIsOurPSF(psf3); + this.VerifyIsOurPSF(psf1); + this.VerifyIsOurPSF(psf2); + this.VerifyIsOurPSF(psf3); querySettings ??= DefaultQuerySettings; return new QueryRecordIterator(psf1, this.QueryPSF(psf1, keys1, querySettings), psf2, this.QueryPSF(psf2, keys2, querySettings), @@ -310,7 +311,7 @@ public IEnumerable QueryPSF( // Power user versions. Anything more complicated than this the caller can post-process with LINQ. internal IEnumerable QueryPSF( - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey : struct @@ -323,8 +324,8 @@ internal IEnumerable QueryPSF( } internal IEnumerable QueryPSF( - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys1, - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct @@ -340,9 +341,9 @@ internal IEnumerable QueryPSF( } internal IEnumerable QueryPSF( - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys1, - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys2, - IEnumerable<(PSF psf, IEnumerable keys)> psfsAndKeys3, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys3, Func matchPredicate, PSFQuerySettings querySettings = null) where TPSFKey1 : struct From 7375d45fcb29a95230d115b420dbcdd7270cb78d Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Wed, 15 Jul 2020 11:54:13 -0700 Subject: [PATCH 12/19] PSF Updates: - Make PSFRegistrationSettings required and move to the first parameter, to allow adding subsequent PSF specs as "params" args (which eliminated one RegisterPSF overload) - Add initial PSF Checkpoint/Restore (more work needed to ensure consistency) - Create directory if needed for LocalStorageDevice - Remove no-longer-needed saving of FasterKV ctor args for PSF --- cs/playground/FasterPSFSample/FPSF.cs | 36 ++++++------- cs/src/core/Device/LocalStorageDevice.cs | 15 ++++-- cs/src/core/Index/FASTER/FASTER.cs | 41 +++++++------- cs/src/core/Index/Interfaces/IFasterKV.cs | 54 ++++++------------- .../core/Index/PSF/FasterPSFRegistration.cs | 54 +++++-------------- cs/src/core/Index/PSF/IExecutePSF.cs | 27 ++++++++++ cs/src/core/Index/PSF/PSFGroup.cs | 23 ++++++-- cs/src/core/Index/PSF/PSFManager.cs | 53 +++++++++++++----- 8 files changed, 169 insertions(+), 134 deletions(-) diff --git a/cs/playground/FasterPSFSample/FPSF.cs b/cs/playground/FasterPSFSample/FPSF.cs index 217a63f0f..2e8addb9c 100644 --- a/cs/playground/FasterPSFSample/FPSF.cs +++ b/cs/playground/FasterPSFSample/FPSF.cs @@ -33,37 +33,37 @@ internal FPSF(bool useObjectValues, bool useMultiGroup, bool useReadCache) if (useMultiGroup) { - var psfOrdinal = 0; - this.SizePsf = FasterKV.RegisterPSF(nameof(this.SizePsf), (k, v) => new SizeKey((Constants.Size)v.SizeInt), - CreatePSFRegistrationSettings(psfOrdinal++)); - this.ColorPsf = FasterKV.RegisterPSF(nameof(this.ColorPsf), (k, v) => new ColorKey(Constants.ColorDict[v.ColorArgb]), - CreatePSFRegistrationSettings(psfOrdinal++)); - this.CountBinPsf = FasterKV.RegisterPSF(nameof(this.CountBinPsf), (k, v) => CountBinKey.GetBin(v.Count, out int bin) - ? new CountBinKey(bin) : (CountBinKey?)null, - CreatePSFRegistrationSettings(psfOrdinal++)); + var groupOrdinal = 0; + this.SizePsf = FasterKV.RegisterPSF(CreatePSFRegistrationSettings(groupOrdinal++), nameof(this.SizePsf), + (k, v) => new SizeKey((Constants.Size)v.SizeInt)); + this.ColorPsf = FasterKV.RegisterPSF(CreatePSFRegistrationSettings(groupOrdinal++), nameof(this.ColorPsf), + (k, v) => new ColorKey(Constants.ColorDict[v.ColorArgb])); + this.CountBinPsf = FasterKV.RegisterPSF(CreatePSFRegistrationSettings(groupOrdinal++), nameof(this.CountBinPsf), + (k, v) => CountBinKey.GetBin(v.Count, out int bin) ? new CountBinKey(bin) : (CountBinKey?)null); } else { - var psfs = FasterKV.RegisterPSF(new(string, Func)[] { - (nameof(this.SizePsf), (k, v) => new CombinedKey((Constants.Size)v.SizeInt)), - (nameof(this.ColorPsf), (k, v) => new CombinedKey(Constants.ColorDict[v.ColorArgb])), - (nameof(this.CountBinPsf), (k, v) => CountBinKey.GetBin(v.Count, out int bin) - ? new CombinedKey(bin) : (CombinedKey?)null) - }, CreatePSFRegistrationSettings(0) - ); + var psfs = FasterKV.RegisterPSF(CreatePSFRegistrationSettings(0), + new (string, Func)[] + { + (nameof(this.SizePsf), (k, v) => new CombinedKey((Constants.Size)v.SizeInt)), + (nameof(this.ColorPsf), (k, v) => new CombinedKey(Constants.ColorDict[v.ColorArgb])), + (nameof(this.CountBinPsf), (k, v) => CountBinKey.GetBin(v.Count, out int bin) + ? new CombinedKey(bin) : (CombinedKey?)null) + }); this.CombinedSizePsf = psfs[0]; this.CombinedColorPsf = psfs[1]; this.CombinedCountBinPsf = psfs[2]; } } - PSFRegistrationSettings CreatePSFRegistrationSettings(int psfOrdinal) + PSFRegistrationSettings CreatePSFRegistrationSettings(int groupOrdinal) { var regSettings = new PSFRegistrationSettings { HashTableSize = 1L << 20, - LogSettings = this.logFiles.PSFLogSettings[psfOrdinal], - CheckpointSettings = null, // TODO checkpoints + LogSettings = this.logFiles.PSFLogSettings[groupOrdinal], + CheckpointSettings = new CheckpointSettings(), // TODO checkpoints IPU1CacheSize = 0, // TODO IPUCache IPU2CacheSize = 0 }; diff --git a/cs/src/core/Device/LocalStorageDevice.cs b/cs/src/core/Device/LocalStorageDevice.cs index 7d3ef2d92..8f2052212 100644 --- a/cs/src/core/Device/LocalStorageDevice.cs +++ b/cs/src/core/Device/LocalStorageDevice.cs @@ -250,16 +250,25 @@ protected internal static SafeFileHandle CreateHandle(int segmentId, bool disabl if (disableFileBuffering) { - fileFlags = fileFlags | Native32.FILE_FLAG_NO_BUFFERING; + fileFlags |= Native32.FILE_FLAG_NO_BUFFERING; } if (deleteOnClose) { - fileFlags = fileFlags | Native32.FILE_FLAG_DELETE_ON_CLOSE; + fileFlags |= Native32.FILE_FLAG_DELETE_ON_CLOSE; // FILE_SHARE_DELETE allows multiple FASTER instances to share a single log directory and each can specify deleteOnClose. // This will allow the files to persist until all handles across all instances have been closed. - fileShare = fileShare | Native32.FILE_SHARE_DELETE; + fileShare |= Native32.FILE_SHARE_DELETE; + } + + var dir = Path.GetDirectoryName(fileName); + try + { + Directory.CreateDirectory(dir); + } catch (Exception ex) + { + throw new IOException($"Error creating log directory for {GetSegmentName(fileName, segmentId)}, error: {ex.Message}", ex); } var logHandle = Native32.CreateFileW( diff --git a/cs/src/core/Index/FASTER/FASTER.cs b/cs/src/core/Index/FASTER/FASTER.cs index d99f1e069..20baaa7e0 100644 --- a/cs/src/core/Index/FASTER/FASTER.cs +++ b/cs/src/core/Index/FASTER/FASTER.cs @@ -184,13 +184,7 @@ public FasterKV(long size, Functions functions, LogSettings logSettings, _systemState.phase = Phase.REST; _systemState.version = 1; - // To pass along to PSF-implementing secondary FasterKV - this.hashTableSize = size; - this.logSettings = logSettings; - this.checkpointSettings = checkpointSettings; - this.serializerSettings = serializerSettings; - this.variableLengthStructSettings = variableLengthStructSettings; - this.InitializePSFs(); + this.InitializePSFManager(); } /// @@ -205,14 +199,15 @@ public FasterKV(long size, Functions functions, LogSettings logSettings, /// public bool TakeFullCheckpoint(out Guid token, long targetVersion = -1) { - ISynchronizationTask backend; - if (FoldOverSnapshot) - backend = new FoldOverCheckpointTask(); - else - backend = new SnapshotCheckpointTask(); + var backend = FoldOverSnapshot ? (ISynchronizationTask)new FoldOverCheckpointTask() : new SnapshotCheckpointTask(); var result = StartStateMachine(new FullCheckpointStateMachine(backend, targetVersion)); token = _hybridLogCheckpointToken; + + // Do not return the PSF token here. TODO Separate Tasks? + if (result && this.PSFManager.HasPSFs) + result &= this.PSFManager.TakeFullCheckpoint(); + return result; } @@ -225,6 +220,10 @@ public bool TakeIndexCheckpoint(out Guid token) { var result = StartStateMachine(new IndexSnapshotStateMachine()); token = _indexCheckpointToken; + + // Do not return the PSF token here. TODO Separate Tasks? + if (result && this.PSFManager.HasPSFs) + result &= this.PSFManager.TakeIndexCheckpoint(); return result; } @@ -236,14 +235,14 @@ public bool TakeIndexCheckpoint(out Guid token) /// Whether we could initiate the checkpoint public bool TakeHybridLogCheckpoint(out Guid token, long targetVersion = -1) { - ISynchronizationTask backend; - if (FoldOverSnapshot) - backend = new FoldOverCheckpointTask(); - else - backend = new SnapshotCheckpointTask(); + var backend = FoldOverSnapshot ? (ISynchronizationTask)new FoldOverCheckpointTask() : new SnapshotCheckpointTask(); var result = StartStateMachine(new HybridLogCheckpointStateMachine(backend, targetVersion)); token = _hybridLogCheckpointToken; + + // Do not return the PSF token here. TODO Separate Tasks? + if (result && this.PSFManager.HasPSFs) + result &= this.PSFManager.TakeHybridLogCheckpoint(); return result; } @@ -253,6 +252,9 @@ public bool TakeHybridLogCheckpoint(out Guid token, long targetVersion = -1) public void Recover() { InternalRecoverFromLatestCheckpoints(); + + if (this.PSFManager.HasPSFs) // TODO Separate Tasks? + this.PSFManager.Recover(); } /// @@ -290,10 +292,13 @@ public async ValueTask CompleteCheckpointAsync(CancellationToken token = default var systemState = _systemState; if (systemState.phase == Phase.REST || systemState.phase == Phase.PREPARE_GROW || systemState.phase == Phase.IN_PROGRESS_GROW) - return; + break; await ThreadStateMachineStep(null, null, true, token); } + + if (this.PSFManager.HasPSFs) // TODO Separate Task? + await this.PSFManager.CompleteCheckpointAsync(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/cs/src/core/Index/Interfaces/IFasterKV.cs b/cs/src/core/Index/Interfaces/IFasterKV.cs index 072e7594d..b51b3eee1 100644 --- a/cs/src/core/Index/Interfaces/IFasterKV.cs +++ b/cs/src/core/Index/Interfaces/IFasterKV.cs @@ -137,15 +137,14 @@ public interface IFasterKV : IDis /// /// static TPSFKey? sizePsfFunc(ref TKVKey key, ref TKVValue value) => new TPSFKey(value.size); /// var sizePsfDef = new FasterKVPSFDefinition{TKVKey, TKVValue, TPSFKey}("sizePSF", sizePsfFunc); - /// var sizePsf = fht.RegisterPSF(sizePsfDef); + /// var sizePsf = fht.RegisterPSF(psfRegistrationSettings, sizePsfDef); /// /// The type of the key value returned from the + /// Registration settings for the secondary FasterKV instances, etc. /// A FasterKV-specific form of a PSF definition - /// Optional registration settings for the secondary FasterKV instances, etc. /// A FasterKV-specific PSF implementation whose TRecordId is long( - IPSF RegisterPSF( - FasterKVPSFDefinition def, - PSFRegistrationSettings registrationSettings = null) + IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, + FasterKVPSFDefinition def) where TPSFKey : struct; /// @@ -154,70 +153,49 @@ IPSF RegisterPSF( /// /// static TPSFKey? sizePsfFunc(ref TKVKey key, ref TKVValue value) => new TPSFKey(value.size); /// var sizePsfDef = new FasterKVPSFDefinition{TKVKey, TKVValue, TPSFKey}("sizePSF", sizePsfFunc); - /// var sizePsf = fht.RegisterPSF(new [] { sizePsfDef }); + /// var sizePsf = fht.RegisterPSF(psfRegistrationSettings, new [] { sizePsfDef }); /// /// The type of the key value returned from the + /// Registration settings for the secondary FasterKV instances, etc. /// An array of FasterKV-specific forms of PSF definitions - /// Optional registration settings for the secondary FasterKV instances, etc. /// A FasterKV-specific PSF implementation whose TRecordId is long( - IPSF[] RegisterPSF - (FasterKVPSFDefinition[] defs, - PSFRegistrationSettings registrationSettings = null) + IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, + params FasterKVPSFDefinition[] defs) where TPSFKey : struct; /// /// Register a with a simple definition. /// /// - /// var sizePsf = fht.RegisterPSF("sizePsf", (k, v) => new TPSFKey(v.size)); + /// var sizePsf = fht.RegisterPSF(psfRegistrationSettings, "sizePsf", (k, v) => new TPSFKey(v.size)); /// /// The type of the key value returned from the + /// Registration settings for the secondary FasterKV instances, etc. /// The name of the PSF; must be unique across all PSFGroups in this FasterKV instance /// A Func implementing the PSF, it will be wrapped in a delegate - /// Optional registration settings for the secondary FasterKV instances, etc. /// A FasterKV-specific implementation whose TRecordId is long - IPSF RegisterPSF( - string psfName, Func psfFunc, - PSFRegistrationSettings registrationSettings = null) + IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, + string psfName, Func psfFunc) where TPSFKey : struct; /// /// Register multiple with no registration settings. /// /// - /// var sizePsf = fht.RegisterPSF(("sizePsf", (k, v) => new TPSFKey(v.size)), + /// var sizePsf = fht.RegisterPSF(psfRegistrationSettings, + /// ("sizePsf", (k, v) => new TPSFKey(v.size)), /// ("colorPsf", (k, v) => new TPSFKey(v.color))); /// /// The type of the key value returned from the + /// Registration settings for the secondary FasterKV instances, etc. /// One or more tuples containing a PSF name and implementing Func; the name must be /// unique across all PSFGroups in this FasterKV instance, and the Func will be wrapped in a delegate /// "params" won't allow the optional fromAddress and keyComparer, so an overload is provided /// to specify those - IPSF[] RegisterPSF( + IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, params (string, Func)[] psfFuncs) where TPSFKey : struct; - /// - /// Register multiple s with registration settings. - /// - /// - /// // Unfortunately the array type cannot be implicitly deduced in current versions of the compiler - /// var sizePsf = fht.RegisterPSF(new (string, Func{TKVKey, TKVValue, TPSFKey})[] { - /// ("sizePsf", (k, v) => new TPSFKey(v.size)), - /// ("colorPsf", (k, v) => new TPSFKey(v.color))}, - /// registrationSettings); - /// - /// The type of the key value returned from the - /// One or more tuples containing a PSF name and implementing Func; the name must be - /// unique across all PSFGroups in this FasterKV instance, and the Func will be wrapped in a delegate - /// Optional registration settings for the secondary FasterKV instances, etc. - /// If the registrationSettings parameters are null, then it is simpler to call the "params" overload - /// than to create the vector explicitly - IPSF[] RegisterPSF( - (string, Func)[] psfDefs, - PSFRegistrationSettings registrationSettings = null) - where TPSFKey : struct; - /// /// Returns the names of registered s for use in recovery. /// TODO: Supplement or replace this with an app version string. diff --git a/cs/src/core/Index/PSF/FasterPSFRegistration.cs b/cs/src/core/Index/PSF/FasterPSFRegistration.cs index 1a4fa5085..8b79a4af2 100644 --- a/cs/src/core/Index/PSF/FasterPSFRegistration.cs +++ b/cs/src/core/Index/PSF/FasterPSFRegistration.cs @@ -15,63 +15,35 @@ public partial class FasterKV where Value : new() where Functions : IFunctions { - // Some FasterKV ctor params we need to pass through. - private readonly long hashTableSize; - private readonly LogSettings logSettings; - private readonly CheckpointSettings checkpointSettings; - private readonly SerializerSettings serializerSettings; - private readonly VariableLengthStructSettings variableLengthStructSettings; - internal PSFManager, long> PSFManager { get; private set; } - internal void InitializePSFs() + internal void InitializePSFManager() => this.PSFManager = new PSFManager, long>(); - private PSFRegistrationSettings CreateDefaultRegistrationSettings() - => new PSFRegistrationSettings - { - HashTableSize = this.hashTableSize, - LogSettings = this.logSettings, - CheckpointSettings = this.checkpointSettings // TODO fix this up - }; - #region PSF Registration API /// - public IPSF RegisterPSF( - FasterKVPSFDefinition def, - PSFRegistrationSettings registrationSettings = null) - where TPSFKey : struct - => this.PSFManager.RegisterPSF(def, registrationSettings ?? CreateDefaultRegistrationSettings()); - - /// - public IPSF[] RegisterPSF - (FasterKVPSFDefinition[] defs, - PSFRegistrationSettings registrationSettings = null) + public IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, + FasterKVPSFDefinition def) where TPSFKey : struct - => this.PSFManager.RegisterPSF(defs, registrationSettings ?? CreateDefaultRegistrationSettings()); + => this.PSFManager.RegisterPSF(registrationSettings, def); /// - public IPSF RegisterPSF( - string psfName, Func psfFunc, - PSFRegistrationSettings registrationSettings = null) + public IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, + params FasterKVPSFDefinition[] defs) where TPSFKey : struct - => this.PSFManager.RegisterPSF(new FasterKVPSFDefinition(psfName, psfFunc), - registrationSettings ?? CreateDefaultRegistrationSettings()); + => this.PSFManager.RegisterPSF(registrationSettings, defs); /// - public IPSF[] RegisterPSF( - params (string, Func)[] psfFuncs) + public IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, + string psfName, Func psfFunc) where TPSFKey : struct - => this.PSFManager.RegisterPSF(psfFuncs.Select(e => new FasterKVPSFDefinition(e.Item1, e.Item2)).ToArray(), - CreateDefaultRegistrationSettings()); + => this.PSFManager.RegisterPSF(registrationSettings, new FasterKVPSFDefinition(psfName, psfFunc)); /// - public IPSF[] RegisterPSF( - (string, Func)[] psfDefs, - PSFRegistrationSettings registrationSettings = null) + public IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, + params (string, Func)[] psfFuncs) where TPSFKey : struct - => this.PSFManager.RegisterPSF(psfDefs.Select(e => new FasterKVPSFDefinition(e.Item1, e.Item2)).ToArray(), - CreateDefaultRegistrationSettings()); + => this.PSFManager.RegisterPSF(registrationSettings, psfFuncs.Select(e => new FasterKVPSFDefinition(e.Item1, e.Item2)).ToArray()); /// public string[][] GetRegisteredPSFNames() => this.PSFManager.GetRegisteredPSFNames(); diff --git a/cs/src/core/Index/PSF/IExecutePSF.cs b/cs/src/core/Index/PSF/IExecutePSF.cs index b73869732..e17d01c8c 100644 --- a/cs/src/core/Index/PSF/IExecutePSF.cs +++ b/cs/src/core/Index/PSF/IExecutePSF.cs @@ -2,6 +2,8 @@ // Licensed under the MIT license. using System; +using System.Threading; +using System.Threading.Tasks; namespace FASTER.core { @@ -49,5 +51,30 @@ Status ExecuteAndStore(TProviderData data, TRecordId recordId, PSFExecutePhase p /// The record of previous key values and updated values /// Status Delete(PSFChangeTracker changeTracker); + + /// + /// Take a full checkpoint of the FasterKV implementing the group's PSFs. + /// + bool TakeFullCheckpoint(); + + /// + /// Complete ongoing checkpoint (spin-wait) + /// + ValueTask CompleteCheckpointAsync(CancellationToken token = default); + + /// + /// Take a checkpoint of the Index (hashtable) only + /// + bool TakeIndexCheckpoint(); + + /// + /// Take a checkpoint of the hybrid log only + /// + bool TakeHybridLogCheckpoint(); + + /// + /// Recover from last successful checkpoints + /// + void Recover(); } } diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index 65c08cbb7..43cd99491 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; namespace FASTER.core { @@ -76,11 +78,11 @@ PSFOutputSecondary, PSFContext, PSFFunctions /// Constructor /// + /// Optional registration settings + /// PSF definitions /// The ordinal of this PSFGroup in the 's /// PSFGroup list. - /// PSF definitions - /// Optional registration settings - public PSFGroup(long id, IPSFDefinition[] defs, PSFRegistrationSettings regSettings) + public PSFGroup(PSFRegistrationSettings regSettings, IPSFDefinition[] defs, long id) { this.psfDefinitions = defs; this.Id = id; @@ -400,5 +402,20 @@ private IEnumerable Query(PSFInputSecondary input) input.Dispose(); } } + + /// + public bool TakeFullCheckpoint() => this.fht.TakeFullCheckpoint(out _); + + /// + public ValueTask CompleteCheckpointAsync(CancellationToken token = default) => this.fht.CompleteCheckpointAsync(token); + + /// + public bool TakeIndexCheckpoint() => this.fht.TakeIndexCheckpoint(out _); + + /// + public bool TakeHybridLogCheckpoint() => this.fht.TakeHybridLogCheckpoint(out _); + + /// + public void Recover() => this.fht.Recover(); } } diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index 3b3dcc397..36f3c2314 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using System.Threading.Tasks; // TODO: Remove PackageId and PackageOutputPath from csproj when this is folded into master @@ -144,11 +145,21 @@ private void VerifyIsOurPSF(IEnumerable<(IPSF, IEnumerable)> p this.VerifyIsOurPSF(psfAndKeys.Item1); } - internal IPSF RegisterPSF(IPSFDefinition def, - PSFRegistrationSettings registrationSettings) + private static void VerifyRegistrationSettings(PSFRegistrationSettings registrationSettings) where TPSFKey : struct + { + if (registrationSettings is null) + throw new PSFArgumentException("PSFRegistrationSettings is required"); + if (registrationSettings.LogSettings is null) + throw new PSFArgumentException("PSFRegistrationSettings.LogSettings is required"); + if (registrationSettings.CheckpointSettings is null) + throw new PSFArgumentException("PSFRegistrationSettings.CheckpointSettings is required"); + } + + internal IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, IPSFDefinition def) where TPSFKey : struct { this.VerifyIsBlittable(); + VerifyRegistrationSettings(registrationSettings); if (def is null) throw new PSFArgumentException("PSF definition cannot be null"); @@ -158,7 +169,7 @@ internal IPSF RegisterPSF(IPSFDefinition def, { if (psfNames.ContainsKey(def.Name)) throw new PSFArgumentException($"A PSF named {def.Name} is already registered in another group"); - var group = new PSFGroup(this.psfGroups.Count, new[] { def }, registrationSettings); + var group = new PSFGroup(registrationSettings, new[] { def }, this.psfGroups.Count); AddGroup(group); var psf = group[def.Name]; this.psfNames.TryAdd(psf.Name, psf.Id); @@ -166,19 +177,13 @@ internal IPSF RegisterPSF(IPSFDefinition def, } } - internal IPSF[] RegisterPSF(IPSFDefinition[] defs, - PSFRegistrationSettings registrationSettings) + internal IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, IPSFDefinition[] defs) where TPSFKey : struct { this.VerifyIsBlittable(); + VerifyRegistrationSettings(registrationSettings); if (defs is null || defs.Length == 0 || defs.Any(def => def is null) || defs.Length == 0) - throw new PSFArgumentException("PSF definitions cannot be null"); - - // For PSFs defined on a FasterKV instance we create intelligent defaults in regSettings. - if (registrationSettings is null) - throw new PSFArgumentException("PSFRegistrationSettings is required"); - if (registrationSettings.LogSettings is null) - throw new PSFArgumentException("PSFRegistrationSettings.LogSettings is required"); + throw new PSFArgumentException("PSF definitions cannot be null or empty"); // This is a very rare operation and unlikely to have any contention, and locking the dictionary // makes it much easier to recover from duplicates if needed. @@ -196,7 +201,7 @@ internal IPSF[] RegisterPSF(IPSFDefinition[] de } } - var group = new PSFGroup(this.psfGroups.Count, defs, registrationSettings); + var group = new PSFGroup(registrationSettings, defs, this.psfGroups.Count); AddGroup(group); foreach (var psf in group.PSFs) this.psfNames.TryAdd(psf.Name, psf.Id); @@ -360,5 +365,27 @@ internal IEnumerable QueryPSF( psfsAndKeys3.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings)))}, matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1], matchIndicators[2]), querySettings).Run(); } + + // TODO Separate Tasks for each group's commit/restore operations? + public bool TakeFullCheckpoint() + => this.psfGroups.Values.Aggregate(true, (result, group) => group.TakeFullCheckpoint() && result); + + public Task CompleteCheckpointAsync(CancellationToken token = default) + { + var tasks = this.psfGroups.Values.Select(group => group.CompleteCheckpointAsync(token).AsTask()).ToArray(); + return Task.WhenAll(tasks); + } + + public bool TakeIndexCheckpoint() + => this.psfGroups.Values.Aggregate(true, (result, group) => group.TakeIndexCheckpoint() && result); + + public bool TakeHybridLogCheckpoint() + => this.psfGroups.Values.Aggregate(true, (result, group) => group.TakeHybridLogCheckpoint() && result); + + public void Recover() + { + foreach (var group in this.psfGroups.Values) + group.Recover(); + } } } From ca9e43c497ee7d1cf81d9f4584c7339db66f704b Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Wed, 12 Aug 2020 11:53:16 -0700 Subject: [PATCH 13/19] Add QueryPSFAsync and flushing PSF logs --- .../FasterPSFSample/BlittableOrders.cs | 13 +- .../FasterPSFSample/FasterPSFSample.cs | 282 ++++++++----- cs/playground/FasterPSFSample/ObjectOrders.cs | 13 +- cs/playground/FasterPSFSample/ParseArgs.cs | 8 + cs/src/core/Allocator/AllocatorBase.cs | 11 +- cs/src/core/ClientSession/FASTERAsync.cs | 56 +-- cs/src/core/Index/FASTER/FASTERImpl.cs | 73 ++-- cs/src/core/Index/FASTER/FASTERThread.cs | 31 +- cs/src/core/Index/Interfaces/IFasterKV.cs | 24 ++ cs/src/core/Index/PSF/DeadRecords.cs | 47 +++ .../Index/PSF/FasterPSFContextOperations.cs | 63 ++- cs/src/core/Index/PSF/FasterPSFImpl.cs | 93 ++--- .../core/Index/PSF/FasterPSFLogOperations.cs | 25 ++ .../Index/PSF/FasterPSFSessionOperations.cs | 365 ++++++++++++++++- cs/src/core/Index/PSF/IExecutePSF.cs | 19 + cs/src/core/Index/PSF/IQueryPSF.cs | 14 +- cs/src/core/Index/PSF/KeyAccessor.cs | 43 +- cs/src/core/Index/PSF/PSF.cs | 11 +- cs/src/core/Index/PSF/PSFGroup.cs | 109 +++-- cs/src/core/Index/PSF/PSFManager.cs | 288 ++++++++++++-- cs/src/core/Index/PSF/PSFQuerySettings.cs | 3 + cs/src/core/Index/PSF/RecordIterator.cs | 371 +++++++++++++----- 22 files changed, 1528 insertions(+), 434 deletions(-) create mode 100644 cs/src/core/Index/PSF/DeadRecords.cs create mode 100644 cs/src/core/Index/PSF/FasterPSFLogOperations.cs diff --git a/cs/playground/FasterPSFSample/BlittableOrders.cs b/cs/playground/FasterPSFSample/BlittableOrders.cs index ffb670b9e..c4111a3b7 100644 --- a/cs/playground/FasterPSFSample/BlittableOrders.cs +++ b/cs/playground/FasterPSFSample/BlittableOrders.cs @@ -29,10 +29,7 @@ public void SingleReader(ref Key key, ref Input input, ref Blit => dst.Value = value; public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) - { - if (((IOrders)output.Value).MemberTuple != key.MemberTuple) - throw new Exception("Read mismatch error!"); - } + { /* Output is not set by pending operations */ } #endregion Read #region Upsert @@ -46,7 +43,7 @@ public void SingleWriter(ref Key key, ref BlittableOrders src, ref BlittableOrde => dst = src; public void UpsertCompletionCallback(ref Key key, ref BlittableOrders value, Context context) - => throw new NotImplementedException(); + { } #endregion Upsert #region RMW @@ -63,14 +60,14 @@ public bool InPlaceUpdater(ref Key key, ref Input input, ref Bl } public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) - => throw new NotImplementedException(); + { } #endregion RMW public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) - => throw new NotImplementedException(); + { } public void DeleteCompletionCallback(ref Key key, Context context) - => throw new NotImplementedException(); + { } } } } diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs index 00c2252b4..53f02becc 100644 --- a/cs/playground/FasterPSFSample/FasterPSFSample.cs +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -10,6 +10,7 @@ using System.Drawing; using System.Linq; using System.Threading; +using System.Threading.Tasks; namespace FasterPSFSample { @@ -31,39 +32,40 @@ public partial class FasterPSFSample internal static int serialNo; - static void Main(string[] argv) + static async Task Main(string[] argv) { if (!ParseArgs(argv)) return; if (useObjectValue) // TODO add VarLenValue - RunSample, Output, ObjectOrders.Functions, ObjectOrders.Serializer>(); + await RunSample, Output, ObjectOrders.Functions, ObjectOrders.Serializer>(); else - RunSample, Output, BlittableOrders.Functions, NoSerializer>(); + await RunSample, Output, BlittableOrders.Functions, NoSerializer>(); return; } - internal static void RunSample() + internal async static Task RunSample() where TValue : IOrders, new() where TInput : IInput, new() where TOutput : IOutput, new() where TFunctions : IFunctions>, new() where TSerializer : BinaryObjectSerializer, new() { - var fpsf = new FPSF(useObjectValue, useMultiGroups, useReadCache: true); + var fpsf = new FPSF(useObjectValue, useMultiGroups, useReadCache: false); // ReadCache and CopyReadsToTail are not supported for PSFs try { CountBinKey.WantLastBin = false; - RunInitialInserts(fpsf); + await RunInitialInserts(fpsf); CountBinKey.WantLastBin = true; - RunReads(fpsf); - var ok = QueryPSFsWithoutBoolOps(fpsf) - && QueryPSFsWithBoolOps(fpsf) - && UpdateSizeByUpsert(fpsf) - && UpdateColorByRMW(fpsf) - && UpdateCountByUpsert(fpsf) - && Delete(fpsf); + await RunReads(fpsf); + var ok = await QueryPSFsWithoutBoolOps(fpsf) + && await QueryPSFsWithBoolOps(fpsf) + && await UpdateSizeByUpsert(fpsf) + && await UpdateColorByRMW(fpsf) + && await UpdateCountByUpsert(fpsf) + && await Delete(fpsf); Console.WriteLine("--------------------------------------------------------"); + Console.WriteLine($"Completed run: UseObjects {useObjectValue}, MultiGroup {useMultiGroups}, Async {useAsync}"); Console.WriteLine(ok ? "Passed! All operations succeeded" : "*** Failed! *** One or more operations did not succeed"); Console.WriteLine(); @@ -81,7 +83,7 @@ internal static void RunSample [Conditional("PSF_TRACE")] static void PsfTraceLine(string message) => Console.WriteLine(message); - internal static void RunInitialInserts(FPSF fpsf) + internal async static Task RunInitialInserts(FPSF fpsf) where TValue : IOrders, new() where TInput : IInput, new() where TOutput : IOutput, new() @@ -157,12 +159,18 @@ internal static void RunInitialInserts(FPSF fpsf) + internal async static Task RunReads(FPSF fpsf) where TValue : IOrders, new() where TInput : IInput, new() where TOutput : IOutput, new() @@ -200,7 +208,15 @@ internal static void RunReads( for (int i = 0; i < UpsertCount; i++) { var key = keys[rng.Next(keys.Length)]; - var status = session.Read(ref key, ref input, ref output, context, serialNo); + var status = Status.OK; + if (useAsync) + { + (status, output) = (await session.ReadAsync(ref key, ref input, context)).CompleteRead(); + } + else + { + session.Read(ref key, ref input, ref output, context, serialNo); + } if (status == Status.OK && output.Value.MemberTuple != key.MemberTuple) throw new Exception($"Error: Value does not match key in {nameof(RunReads)}"); @@ -232,7 +248,7 @@ static bool VerifyProviderDatas(FasterKVProviderData[] prov return expectedCount == providerDatas.Length; } - internal static bool QueryPSFsWithoutBoolOps(FPSF fpsf) + internal async static Task QueryPSFsWithoutBoolOps(FPSF fpsf) where TValue : IOrders, new() where TInput : IInput, new() where TOutput : IOutput, new() @@ -246,29 +262,34 @@ internal static bool QueryPSFsWithoutBoolOps[] providerDatas = null; var ok = true; + async Task[]> RunQuery(IPSF psf, TPSFKey key) where TPSFKey : struct + => useAsync + ? await session.QueryPSFAsync(psf, key).ToArrayAsync() + : session.QueryPSF(psf, key).ToArray(); + providerDatas = useMultiGroups - ? session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium)).ToArray() - : session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium)).ToArray(); + ? await RunQuery(fpsf.SizePsf, new SizeKey(Constants.Size.Medium)) + : await RunQuery(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium)); ok &= VerifyProviderDatas(providerDatas, "Medium", mediumCount); providerDatas = useMultiGroups - ? session.QueryPSF(fpsf.ColorPsf, new ColorKey(Color.Blue)).ToArray() - : session.QueryPSF(fpsf.CombinedColorPsf, new CombinedKey(Color.Blue)).ToArray(); + ? await RunQuery(fpsf.ColorPsf, new ColorKey(Color.Blue)) + : await RunQuery(fpsf.CombinedColorPsf, new CombinedKey(Color.Blue)); ok &= VerifyProviderDatas(providerDatas, "Blue", blueCount); providerDatas = useMultiGroups - ? session.QueryPSF(fpsf.CountBinPsf, new CountBinKey(7)).ToArray() - : session.QueryPSF(fpsf.CombinedCountBinPsf, new CombinedKey(7)).ToArray(); + ? await RunQuery(fpsf.CountBinPsf, new CountBinKey(7)) + : await RunQuery(fpsf.CombinedCountBinPsf, new CombinedKey(7)); ok &= VerifyProviderDatas(providerDatas, "Bin7", bin7Count); providerDatas = useMultiGroups - ? session.QueryPSF(fpsf.CountBinPsf, new CountBinKey(CountBinKey.LastBin)).ToArray() - : session.QueryPSF(fpsf.CombinedCountBinPsf, new CombinedKey(CountBinKey.LastBin)).ToArray(); + ? await RunQuery(fpsf.CountBinPsf, new CountBinKey(CountBinKey.LastBin)) + : await RunQuery(fpsf.CombinedCountBinPsf, new CombinedKey(CountBinKey.LastBin)); ok &= VerifyProviderDatas(providerDatas, "LastBin", 0); // Insert skipped (returned null from the PSF) all that fall into the last bin return ok; } - internal static bool QueryPSFsWithBoolOps(FPSF fpsf) + internal async static Task QueryPSFsWithBoolOps(FPSF fpsf) where TValue : IOrders, new() where TInput : IInput, new() where TOutput : IOutput, new() @@ -282,87 +303,135 @@ internal static bool QueryPSFsWithBoolOps[] providerDatas = null; var ok = true; + // Local functions can't be overloaded so make the name unique + async Task[]> RunQuery2( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + Func matchPredicate) + where TPSFKey1 : struct + where TPSFKey2 : struct + => useAsync + ? await session.QueryPSFAsync(psf1, key1, psf2, key2, matchPredicate).ToArrayAsync() + : session.QueryPSF(psf1, key1, psf2, key2, matchPredicate).ToArray(); + + async Task[]> RunQuery3( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + IPSF psf3, TPSFKey3 key3, + Func matchPredicate) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + => useAsync + ? await session.QueryPSFAsync(psf1, key1, psf2, key2, psf3, key3, matchPredicate).ToArrayAsync() + : session.QueryPSF(psf1, key1, psf2, key2, psf3, key3, matchPredicate).ToArray(); + + async Task[]> RunQuery1EnumKeys( + IPSF psf, IEnumerable keys, PSFQuerySettings querySettings = null) + where TPSFKey : struct + => useAsync + ? await session.QueryPSFAsync(psf, keys).ToArrayAsync() + : session.QueryPSF(psf, keys).ToArray(); + + async Task[]> RunQuery2Vec( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + Func matchPredicate) + where TPSFKey1 : struct + where TPSFKey2 : struct + => useAsync + ? await session.QueryPSFAsync(psf1, keys1, psf2, keys2, matchPredicate).ToArrayAsync() + : session.QueryPSF(psf1, keys1, psf2, keys2, matchPredicate).ToArray(); + + async Task[]> RunQuery1EnumTuple( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, + Func matchPredicate) + where TPSFKey : struct + => useAsync + ? await session.QueryPSFAsync(psfsAndKeys, matchPredicate).ToArrayAsync() + : session.QueryPSF(psfsAndKeys, matchPredicate).ToArray(); + if (useMultiGroups) { - providerDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), - fpsf.ColorPsf, new ColorKey(Color.Blue), (sz, cl) => sz && cl).ToArray(); + providerDatas = await RunQuery2(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), (sz, cl) => sz && cl); ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlueCount), intersectMediumBlueCount); - providerDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), - fpsf.ColorPsf, new ColorKey(Color.Blue), (sz, cl) => sz || cl).ToArray(); + providerDatas = await RunQuery2(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), (sz, cl) => sz || cl); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlueCount), unionMediumBlueCount); - providerDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), - fpsf.ColorPsf, new ColorKey(Color.Blue), - fpsf.CountBinPsf, new CountBinKey(7), (sz, cl, ct) => sz && cl && ct).ToArray(); + providerDatas = await RunQuery3(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), + fpsf.CountBinPsf, new CountBinKey(7), (sz, cl, ct) => sz && cl && ct); ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlue7Count), intersectMediumBlue7Count); - providerDatas = session.QueryPSF(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), - fpsf.ColorPsf, new ColorKey(Color.Blue), - fpsf.CountBinPsf, new CountBinKey(7), (sz, cl, ct) => sz || cl || ct).ToArray(); + providerDatas = await RunQuery3(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), + fpsf.CountBinPsf, new CountBinKey(7), (sz, cl, ct) => sz || cl || ct); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlue7Count), unionMediumBlue7Count); - providerDatas = session.QueryPSF(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }).ToArray(); + providerDatas = await RunQuery1EnumKeys(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeCount), unionMediumLargeCount); - providerDatas = session.QueryPSF(fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }).ToArray(); + providerDatas = await RunQuery1EnumKeys(fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }); ok &= VerifyProviderDatas(providerDatas, nameof(unionRedBlueCount), unionRedBlueCount); - providerDatas = session.QueryPSF(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }, - fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }, (sz, cl) => sz && cl).ToArray(); + providerDatas = await RunQuery2Vec(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }, + fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }, (sz, cl) => sz && cl); ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumLargeRedBlueCount), intersectMediumLargeRedBlueCount); - providerDatas = session.QueryPSF(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }, - fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }, (sz, cl) => sz || cl).ToArray(); + providerDatas = await RunQuery2Vec(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }, + fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }, (sz, cl) => sz || cl); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeRedBlueCount), unionMediumLargeRedBlueCount); } else { - providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), - fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), (sz, cl) => sz && cl).ToArray(); + providerDatas = await RunQuery2(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), (sz, cl) => sz && cl); ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlueCount), intersectMediumBlueCount); - providerDatas = session.QueryPSF(new[] { (fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), - (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue) }.AsEnumerable()) }, sz => sz[0] && sz[1]).ToArray(); + providerDatas = await RunQuery1EnumTuple(new[] { (fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue) }.AsEnumerable()) }, sz => sz[0] && sz[1]); ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlueCount), intersectMediumBlueCount); // --- - providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), - fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), (sz, cl) => sz || cl).ToArray(); + providerDatas = await RunQuery2(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), (sz, cl) => sz || cl); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlueCount), unionMediumBlueCount); - providerDatas = session.QueryPSF(new[] { (fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), - (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue) }.AsEnumerable()) }, sz => sz[0] || sz[1]).ToArray(); + providerDatas = await RunQuery1EnumTuple(new[] { (fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue) }.AsEnumerable()) }, sz => sz[0] || sz[1]); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlueCount), unionMediumBlueCount); // --- - providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), - fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), - fpsf.CombinedCountBinPsf, new CombinedKey(7), (sz, cl, ct) => sz && cl && ct).ToArray(); + providerDatas = await RunQuery3(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), + fpsf.CombinedCountBinPsf, new CombinedKey(7), (sz, cl, ct) => sz && cl && ct); ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlue7Count), intersectMediumBlue7Count); - providerDatas = session.QueryPSF(new[] {(fpsf.CombinedSizePsf, new [] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + providerDatas = await RunQuery1EnumTuple(new[] {(fpsf.CombinedSizePsf, new [] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), (fpsf.CombinedColorPsf, new [] { new CombinedKey(Color.Blue) }.AsEnumerable()), - (fpsf.CombinedCountBinPsf, new [] { new CombinedKey(7) }.AsEnumerable()) }, sz => sz[0] && sz[1] && sz[2]).ToArray(); + (fpsf.CombinedCountBinPsf, new [] { new CombinedKey(7) }.AsEnumerable()) }, sz => sz[0] && sz[1] && sz[2]); ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlue7Count), intersectMediumBlue7Count); // --- - providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), - fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), - fpsf.CombinedCountBinPsf, new CombinedKey(7), (sz, cl, ct) => sz || cl || ct).ToArray(); + providerDatas = await RunQuery3(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), + fpsf.CombinedCountBinPsf, new CombinedKey(7), (sz, cl, ct) => sz || cl || ct); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlue7Count), unionMediumBlue7Count); - providerDatas = session.QueryPSF(new[] {(fpsf.CombinedSizePsf, new [] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + providerDatas = await RunQuery1EnumTuple(new[] {(fpsf.CombinedSizePsf, new [] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), (fpsf.CombinedColorPsf, new [] { new CombinedKey(Color.Blue) }.AsEnumerable()), - (fpsf.CombinedCountBinPsf, new [] { new CombinedKey(7) }.AsEnumerable()) }, sz => sz[0] || sz[1] || sz[2]).ToArray(); + (fpsf.CombinedCountBinPsf, new [] { new CombinedKey(7) }.AsEnumerable()) }, sz => sz[0] || sz[1] || sz[2]); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlue7Count), unionMediumBlue7Count); // --- - providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }).ToArray(); + providerDatas = await RunQuery1EnumKeys(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeCount), unionMediumLargeCount); - providerDatas = session.QueryPSF(fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }).ToArray(); + providerDatas = await RunQuery1EnumKeys(fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }); ok &= VerifyProviderDatas(providerDatas, nameof(unionRedBlueCount), unionRedBlueCount); // --- - providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }, - fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }, (sz, cl) => sz && cl).ToArray(); + providerDatas = await RunQuery2Vec(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }, + fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }, (sz, cl) => sz && cl); ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumLargeRedBlueCount), intersectMediumLargeRedBlueCount); - providerDatas = session.QueryPSF(new[] {(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }.AsEnumerable()), - (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }.AsEnumerable())}, sz => sz[0] && sz[1]).ToArray(); + providerDatas = await RunQuery1EnumTuple(new[] {(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }.AsEnumerable())}, sz => sz[0] && sz[1]); ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumLargeRedBlueCount), intersectMediumLargeRedBlueCount); // --- - providerDatas = session.QueryPSF(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }, - fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }, (sz, cl) => sz || cl).ToArray(); + providerDatas = await RunQuery2Vec(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }, + fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }, (sz, cl) => sz || cl); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeRedBlueCount), unionMediumLargeRedBlueCount); - providerDatas = session.QueryPSF(new[] {(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }.AsEnumerable()), - (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }.AsEnumerable())}, sz => sz[0] || sz[1]).ToArray(); + providerDatas = await RunQuery1EnumTuple(new[] {(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }.AsEnumerable())}, sz => sz[0] || sz[1]); ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeRedBlueCount), unionMediumLargeRedBlueCount); } @@ -377,7 +446,7 @@ private static void WriteResult(bool isInitial, string name, int expectedCount, : $"{indent4}{tag} {name} Failed: expected ({expectedCount}) != actual ({actualCount})"); } - internal static bool UpdateSizeByUpsert(FPSF fpsf) + internal async static Task UpdateSizeByUpsert(FPSF fpsf) where TValue : IOrders, new() where TInput : IInput, new() where TOutput : IOutput, new() @@ -404,21 +473,23 @@ FasterKVProviderData[] GetSizeDatas(Constants.Size size) var context = new Context(); foreach (var providerData in mediumDatas) { - // Get the old value and confirm it's as expected. - ref TValue oldValue = ref providerData.GetValue(); - Debug.Assert(oldValue.SizeInt == (int)Constants.Size.Medium); + // Get the old value and confirm it's as expected. We cannot have ref locals because this is an async function. + Debug.Assert(providerData.GetValue().SizeInt == (int)Constants.Size.Medium); // Clone the old value with updated Size; note that this cannot modify the "ref providerData.GetValue()" in-place as that will bypass PSFs. var newValue = new TValue { - Id = oldValue.Id, + Id = providerData.GetValue().Id, SizeInt = (int)Constants.Size.XXLarge, // updated - ColorArgb = oldValue.ColorArgb, - Count = oldValue.Count + ColorArgb = providerData.GetValue().ColorArgb, + Count = providerData.GetValue().Count }; // Reuse the same key - session.Upsert(ref providerData.GetKey(), ref newValue, context, serialNo); + if (useAsync) + await session.UpsertAsync(ref providerData.GetKey(), ref newValue, context); + else + session.Upsert(ref providerData.GetKey(), ref newValue, context, serialNo); RemoveIfSkippedLastBinKey(ref providerData.GetKey()); } @@ -432,7 +503,7 @@ FasterKVProviderData[] GetSizeDatas(Constants.Size size) return ok; } - internal static bool UpdateColorByRMW(FPSF fpsf) + internal async static Task UpdateColorByRMW(FPSF fpsf) where TValue : IOrders, new() where TInput : IInput, new() where TOutput : IOutput, new() @@ -461,7 +532,11 @@ FasterKVProviderData[] GetColorDatas(Color color) foreach (var providerData in blueDatas) { // This will call Functions<>.InPlaceUpdater. - session.RMW(ref providerData.GetKey(), ref input, context, serialNo); + if (useAsync) + await session.RMWAsync(ref providerData.GetKey(), ref input, context); + else + session.RMW(ref providerData.GetKey(), ref input, context, serialNo); + RemoveIfSkippedLastBinKey(ref providerData.GetKey()); } ++serialNo; @@ -474,7 +549,7 @@ FasterKVProviderData[] GetColorDatas(Color color) return ok; } - internal static bool UpdateCountByUpsert(FPSF fpsf) + internal async static Task UpdateCountByUpsert(FPSF fpsf) where TValue : IOrders, new() where TInput : IInput, new() where TOutput : IOutput, new() @@ -506,22 +581,24 @@ FasterKVProviderData[] GetCountDatas(int bin) var context = new Context(); foreach (var providerData in bin7Datas) { - // Get the old value and confirm it's as expected. - ref TValue oldValue = ref providerData.GetValue(); - Debug.Assert(CountBinKey.GetBin(oldValue.Count, out int tempBin) && tempBin == bin7); + // Get the old value and confirm it's as expected. We cannot have ref locals because this is an async function. + Debug.Assert(CountBinKey.GetBin(providerData.GetValue().Count, out int tempBin) && tempBin == bin7); // Clone the old value with updated Count; note that this cannot modify the "ref providerData.GetValue()" in-place as that will bypass PSFs. var newValue = new TValue { - Id = oldValue.Id, - SizeInt = oldValue.SizeInt, - ColorArgb = oldValue.ColorArgb, - Count = oldValue.Count + (CountBinKey.LastBin - bin7) * CountBinKey.BinSize // updated + Id = providerData.GetValue().Id, + SizeInt = providerData.GetValue().SizeInt, + ColorArgb = providerData.GetValue().ColorArgb, + Count = providerData.GetValue().Count + (CountBinKey.LastBin - bin7) * CountBinKey.BinSize // updated }; Debug.Assert(CountBinKey.GetBin(newValue.Count, out tempBin) && tempBin == CountBinKey.LastBin); // Reuse the same key - session.Upsert(ref providerData.GetKey(), ref newValue, context, serialNo); + if (useAsync) + await session.UpsertAsync(ref providerData.GetKey(), ref newValue, context); + else + session.Upsert(ref providerData.GetKey(), ref newValue, context, serialNo); } ++serialNo; @@ -536,7 +613,7 @@ FasterKVProviderData[] GetCountDatas(int bin) return ok; } - internal static bool Delete(FPSF fpsf) + internal async static Task Delete(FPSF fpsf) where TValue : IOrders, new() where TInput : IInput, new() where TOutput : IOutput, new() @@ -548,12 +625,16 @@ internal static bool Delete(FP using var session = fpsf.FasterKV.NewSession(); - FasterKVProviderData[] GetColorDatas(Color color) + async Task[]> GetColorDatas(Color color) => useMultiGroups - ? session.QueryPSF(fpsf.ColorPsf, new ColorKey(color)).ToArray() - : session.QueryPSF(fpsf.CombinedColorPsf, new CombinedKey(color)).ToArray(); - - var redDatas = GetColorDatas(Color.Red); + ? useAsync + ? await session.QueryPSFAsync(fpsf.ColorPsf, new ColorKey(color)).ToArrayAsync() + : session.QueryPSF(fpsf.ColorPsf, new ColorKey(color)).ToArray() + : useAsync + ? await session.QueryPSFAsync(fpsf.CombinedColorPsf, new CombinedKey(color)).ToArrayAsync() + : session.QueryPSF(fpsf.CombinedColorPsf, new CombinedKey(color)).ToArray(); + + var redDatas = await GetColorDatas(Color.Red); Console.WriteLine(); Console.Write($"Deleting all Reds; initial count {redDatas.Length}"); @@ -561,12 +642,15 @@ FasterKVProviderData[] GetColorDatas(Color color) foreach (var providerData in redDatas) { // This will call Functions<>.InPlaceUpdater. - session.Delete(ref providerData.GetKey(), context, serialNo); + if (useAsync) + await session.DeleteAsync(ref providerData.GetKey(), context); + else + session.Delete(ref providerData.GetKey(), context, serialNo); } ++serialNo; Console.WriteLine(); - redDatas = GetColorDatas(Color.Red); + redDatas = await GetColorDatas(Color.Red); var ok = redDatas.Length == 0; Console.Write(ok ? "Passed" : "*** Failed *** "); Console.WriteLine($": Red {redDatas.Length}"); diff --git a/cs/playground/FasterPSFSample/ObjectOrders.cs b/cs/playground/FasterPSFSample/ObjectOrders.cs index 072f99ddc..c0693546d 100644 --- a/cs/playground/FasterPSFSample/ObjectOrders.cs +++ b/cs/playground/FasterPSFSample/ObjectOrders.cs @@ -54,10 +54,7 @@ public void SingleReader(ref Key key, ref Input input, ref ObjectO => dst.Value = value; public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) - { - if (((IOrders)output.Value).MemberTuple != key.MemberTuple) - throw new Exception("Read mismatch error!"); - } + { /* Output is not set by pending operations */ } #endregion Read #region Upsert @@ -71,7 +68,7 @@ public void SingleWriter(ref Key key, ref ObjectOrders src, ref ObjectOrders dst => dst = src; public void UpsertCompletionCallback(ref Key key, ref ObjectOrders value, Context context) - => throw new NotImplementedException(); + { } #endregion Upsert #region RMW @@ -88,14 +85,14 @@ public bool InPlaceUpdater(ref Key key, ref Input input, ref Objec } public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) - => throw new NotImplementedException(); + { } #endregion RMW public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) - => throw new NotImplementedException(); + { } public void DeleteCompletionCallback(ref Key key, Context context) - => throw new NotImplementedException(); + { } } } } diff --git a/cs/playground/FasterPSFSample/ParseArgs.cs b/cs/playground/FasterPSFSample/ParseArgs.cs index dfa57943f..1cbc704c8 100644 --- a/cs/playground/FasterPSFSample/ParseArgs.cs +++ b/cs/playground/FasterPSFSample/ParseArgs.cs @@ -9,10 +9,12 @@ public partial class FasterPSFSample { private static bool useObjectValue; private static bool useMultiGroups; + private static bool useAsync; private static bool verbose; const string ObjValuesArg = "--objValues"; const string MultiGroupArg = "--multiGroup"; + const string AsyncArg = "--async"; static bool ParseArgs(string[] argv) { @@ -23,6 +25,7 @@ static bool Usage(string message = null) Console.WriteLine(); Console.WriteLine($" {ObjValuesArg}: Use objects instead of blittable Value; default is {useObjectValue}"); Console.WriteLine($" {MultiGroupArg}: Put each PSF in a separate group; default is {useMultiGroups}"); + Console.WriteLine($" {AsyncArg}: Use Async operations on FasterKV; default is {useAsync}"); Console.WriteLine(); if (!string.IsNullOrEmpty(message)) { @@ -47,6 +50,11 @@ static bool Usage(string message = null) useMultiGroups = true; continue; } + if (string.Compare(arg, AsyncArg, ignoreCase: true) == 0) + { + useAsync = true; + continue; + } if (string.Compare(arg, "--help", ignoreCase: true) == 0 || arg == "/?" || arg == "-?") return Usage(); if (string.Compare(arg, "-v", ignoreCase: true) == 0) diff --git a/cs/src/core/Allocator/AllocatorBase.cs b/cs/src/core/Allocator/AllocatorBase.cs index 571e2f9ef..56091d2c7 100644 --- a/cs/src/core/Allocator/AllocatorBase.cs +++ b/cs/src/core/Allocator/AllocatorBase.cs @@ -53,6 +53,8 @@ public unsafe abstract partial class AllocatorBase : IDisposable /// protected readonly IFasterEqualityComparer comparer; + internal IKeyAccessor PsfKeyAccessor; + #region Protected size definitions /// /// Buffer size @@ -1459,9 +1461,14 @@ private void AsyncGetFromDiskCallback(uint errorCode, uint numBytes, NativeOverl // We have the complete record. if (RetrievedFullRecord(record, ref ctx)) { - if (comparer.Equals(ref ctx.request_key.Get(), ref GetContextRecordKey(ref ctx))) + // If the keys are same, then I/O is complete. For PSFs, we may be querying on an address instead of a key; + // in this case, ctx.request_key is null for the primary FKV but we know the address is the one we want. + var isEqualKey = ctx.request_key is null + || (this.PsfKeyAccessor is null + ? comparer.Equals(ref ctx.request_key.Get(), ref GetContextRecordKey(ref ctx)) + : this.PsfKeyAccessor.EqualsAtRecordAddress(ref ctx.request_key.Get(), (long)record)); + if (isEqualKey) { - // The keys are same, so I/O is complete // ctx.record = result.record; if (ctx.callbackQueue != null) ctx.callbackQueue.Enqueue(ctx); diff --git a/cs/src/core/ClientSession/FASTERAsync.cs b/cs/src/core/ClientSession/FASTERAsync.cs index d351cd05d..05573df46 100644 --- a/cs/src/core/ClientSession/FASTERAsync.cs +++ b/cs/src/core/ClientSession/FASTERAsync.cs @@ -229,39 +229,39 @@ internal ValueTask ReadAsync(ClientSession SlowReadAsync( - FasterKV @this, - ClientSession clientSession, - PendingContext pendingContext, CancellationToken token = default) - { - var diskRequest = @this.ScheduleGetFromDisk(clientSession.ctx, ref pendingContext); - clientSession.ctx.ioPendingRequests.Add(pendingContext.id, pendingContext); - clientSession.ctx.asyncPendingCount++; - clientSession.ctx.pendingReads.Add(); + private static async ValueTask SlowReadAsync( + FasterKV @this, + ClientSession clientSession, + PendingContext pendingContext, CancellationToken token = default) + { + var diskRequest = @this.ScheduleGetFromDisk(clientSession.ctx, ref pendingContext); + clientSession.ctx.ioPendingRequests.Add(pendingContext.id, pendingContext); + clientSession.ctx.asyncPendingCount++; + clientSession.ctx.pendingReads.Add(); - try - { - token.ThrowIfCancellationRequested(); + try + { + token.ThrowIfCancellationRequested(); - if (@this.epoch.ThisInstanceProtected()) - throw new NotSupportedException("Async operations not supported over protected epoch"); + if (@this.epoch.ThisInstanceProtected()) + throw new NotSupportedException("Async operations not supported over protected epoch"); - diskRequest = await diskRequest.asyncOperation.ValueTaskOfT; - } - catch - { - clientSession.ctx.ioPendingRequests.Remove(pendingContext.id); - clientSession.ctx.asyncPendingCount--; - throw; - } - finally - { - clientSession.ctx.pendingReads.Remove(); - } - - return new ReadAsyncResult(@this, clientSession, pendingContext, diskRequest); + diskRequest = await diskRequest.asyncOperation.ValueTaskOfT; + } + catch + { + clientSession.ctx.ioPendingRequests.Remove(pendingContext.id); + clientSession.ctx.asyncPendingCount--; + throw; + } + finally + { + clientSession.ctx.pendingReads.Remove(); } + + return new ReadAsyncResult(@this, clientSession, pendingContext, diskRequest); } internal bool AtomicSwitch(FasterExecutionContext fromCtx, FasterExecutionContext toCtx, int version) diff --git a/cs/src/core/Index/FASTER/FASTERImpl.cs b/cs/src/core/Index/FASTER/FASTERImpl.cs index e3e3a4ade..efe280ef8 100644 --- a/cs/src/core/Index/FASTER/FASTERImpl.cs +++ b/cs/src/core/Index/FASTER/FASTERImpl.cs @@ -315,7 +315,7 @@ internal OperationStatus InternalUpsert( logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; TraceBackForKeyMatch(ref key, logicalAddress, - hlog.ReadOnlyAddress, + this.PSFManager.HasPSFs ? hlog.HeadAddress : hlog.ReadOnlyAddress, out logicalAddress, out physicalAddress); } @@ -458,29 +458,23 @@ internal OperationStatus InternalUpsert( if (this.PSFManager.HasPSFs) { psfArgs.LogicalAddress = newLogicalAddress; - var isInsert = logicalAddress < hlog.BeginAddress || - (logicalAddress >= hlog.HeadAddress && hlog.GetInfo(physicalAddress).Tombstone); - if (isInsert) + + if (logicalAddress < hlog.HeadAddress || hlog.GetInfo(physicalAddress).Tombstone) { - // Old logicalAddress is invalid (deos not exist) or was deleted, so this is an insert. - // This goes through the fast Insert path which does not create a changeTracker. + // Old logicalAddress is invalid (LA < BeginAddress, so the record does not exist), on disk (LA < HeadAddress; this + // would require an IO to get it, so instead we defer to the liveness check in PsfInternalReadAddress), or was deleted, + // so this is an insert. This goes through the fast Insert path which does not create a changeTracker. } - else if (logicalAddress >= hlog.HeadAddress) + else { - // The old record was valid but not in mutable range (that's handled above), so this is an RCU + // The old record was valid but not in the mutable range (that's handled above), but it's above HeadAddress, so we can + // get the old value and make this an RCU. + Debug.Assert(logicalAddress < hlog.ReadOnlyAddress); psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: false); SetAfterData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); psfArgs.ChangeTracker.UpdateOp = UpdateOperation.RCU; } - else - { - // ah, old record slipped onto disk - hlog.GetInfo(newPhysicalAddress).Invalid = true; - psfArgs.ChangeTracker = null; - status = OperationStatus.RETRY_NOW; - goto LatchRelease; - } } var updatedEntry = default(HashBucketEntry); @@ -1351,32 +1345,42 @@ internal OperationStatus InternalContinuePendingRead( { Debug.Assert(hlog.GetInfoFromBytePointer(request.record.GetValidPointer()).Version <= ctx.version); - if (hlog.GetInfoFromBytePointer(request.record.GetValidPointer()).Tombstone) - return OperationStatus.NOTFOUND; + var tombstone = hlog.GetInfoFromBytePointer(request.record.GetValidPointer()).Tombstone; + + // For PSF_READ_ADDRESS on the Primary FKV, we have no key on the query, so we have to populate it here + // for the key-match traceback. + if (pendingContext.type == OperationType.PSF_READ_ADDRESS && pendingContext.key is null) + pendingContext.key = hlog.GetKeyContainer(ref hlog.GetContextRecordKey(ref request)); if (pendingContext.type == OperationType.READ) { - functions.SingleReader(ref pendingContext.key.Get(), ref pendingContext.input, - ref hlog.GetContextRecordValue(ref request), ref pendingContext.output); + if (!tombstone) + { + functions.SingleReader(ref pendingContext.key.Get(), ref pendingContext.input, + ref hlog.GetContextRecordValue(ref request), ref pendingContext.output); + } } else if (pendingContext.type == OperationType.PSF_READ_KEY) { pendingContext.psfReadArgs.Output.Visit(pendingContext.psfReadArgs.Input.PsfOrdinal, - ref pendingContext.key.Get(), + ref hlog.GetContextRecordKey(ref request), ref hlog.GetContextRecordValue(ref request), tombstone: false, // checked above isConcurrent: false); } else if (pendingContext.type == OperationType.PSF_READ_ADDRESS) { - var key = default(Key); // Reading by address does not have a key pendingContext.psfReadArgs.Output.Visit(pendingContext.psfReadArgs.Input.PsfOrdinal, - ref key, ref hlog.GetContextRecordValue(ref request), + ref hlog.GetContextRecordKey(ref request), + ref hlog.GetContextRecordValue(ref request), tombstone: false, // checked above isConcurrent: false); } - if (CopyReadsToTail || UseReadCache) + if (tombstone) + return OperationStatus.NOTFOUND; + + if (CopyReadsToTail || UseReadCache && !this.ImplmentsPSFs) // TODOdcr: Support ReadCache and CopyReadsToTail for PSFs { InternalContinuePendingReadCopyToTail(ctx, request, ref pendingContext, currentCtx); } @@ -1852,7 +1856,7 @@ private bool TraceBackForKeyMatch( out long foundLogicalAddress, out long foundPhysicalAddress) { - Debug.Assert(this.psfKeyAccessor is null); + Debug.Assert(!this.ImplmentsPSFs); foundLogicalAddress = fromLogicalAddress; while (foundLogicalAddress >= minOffset) { @@ -2098,25 +2102,6 @@ private bool ReadFromCache(ref Key key, ref long logicalAddress, ref long physic return false; } - private bool ReadFromCache(ref long logicalAddress, ref long physicalAddress, ref int latestRecordVersion) - { - HashBucketEntry entry = default; - entry.word = logicalAddress; - if (!entry.ReadCache) return false; - - physicalAddress = readcache.GetPhysicalAddress(logicalAddress & ~Constants.kReadCacheBitMask); - latestRecordVersion = readcache.GetInfo(physicalAddress).Version; - - if (!readcache.GetInfo(physicalAddress).Invalid) - { - if ((logicalAddress & ~Constants.kReadCacheBitMask) >= readcache.SafeReadOnlyAddress) - return true; - } - - physicalAddress = 0; - return false; - } - private void SkipReadCache(ref long logicalAddress, ref int latestRecordVersion) { HashBucketEntry entry = default; diff --git a/cs/src/core/Index/FASTER/FASTERThread.cs b/cs/src/core/Index/FASTER/FASTERThread.cs index 64bcdfbcb..f2f213d51 100644 --- a/cs/src/core/Index/FASTER/FASTERThread.cs +++ b/cs/src/core/Index/FASTER/FASTERThread.cs @@ -228,12 +228,6 @@ internal void InternalCompleteRetryRequest(FasterExecutionContext opCtx, FasterE internalStatus = InternalDelete(ref key, ref pendingContext.userContext, ref pendingContext, currentCtx, pendingContext.serialNum, ref pendingContext.psfUpdateArgs); break; - case OperationType.PSF_READ_ADDRESS: - // This is a special case for RECORD_ON_DISK in ContextPsfReadAddress during CPR; - // it is the only Read operation that goes through Retry. - internalStatus = this.PsfInternalReadAddress(ref pendingContext.psfReadArgs, ref pendingContext, - currentCtx, pendingContext.serialNum); - break; case OperationType.PSF_READ_KEY: case OperationType.READ: throw new FasterException("Reads go through the Pending route, not Retry, so this cannot happen!"); @@ -345,8 +339,6 @@ internal void InternalCompletePendingRequest(FasterExecutionContext opCtx, Faste { if (opCtx.ioPendingRequests.TryGetValue(request.id, out PendingContext pendingContext)) { - ref Key key = ref pendingContext.key.Get(); - // Remove from pending dictionary opCtx.ioPendingRequests.Remove(request.id); @@ -366,16 +358,12 @@ internal void InternalCompletePendingRequest(FasterExecutionContext opCtx, Faste request.Dispose(); - Status status; - // Handle operation status - if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) - { - status = (Status)internalStatus; - } - else - { - status = HandleOperationStatus(opCtx, currentCtx, pendingContext, internalStatus); - } + Status status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(opCtx, currentCtx, pendingContext, internalStatus); + + // This is set in InternalContinuePendingRead for PSF_READ_ADDRESS, so don't retrieve it until after that. + ref Key key = ref pendingContext.key.Get(); // If done, callback user code if (status == Status.OK || status == Status.NOTFOUND) @@ -391,7 +379,7 @@ internal void InternalCompletePendingRequest(FasterExecutionContext opCtx, Faste pendingContext.userContext, status); } - else + else if (pendingContext.type == OperationType.RMW) { functions.RMWCompletionCallback(ref key, ref pendingContext.input, @@ -407,8 +395,7 @@ internal void InternalCompletePendingRequest(FasterExecutionContext opCtx, Faste { (Status, Output) s = default; - ref Key key = ref pendingContext.key.Get(); - + // PSFs may read by address rather than key, and for the Primary FKV will not have pendingContext.key; this call will fill it in. OperationStatus internalStatus = InternalContinuePendingRead(opCtx, request, ref pendingContext, currentCtx); request.Dispose(); @@ -424,6 +411,8 @@ internal void InternalCompletePendingRequest(FasterExecutionContext opCtx, Faste throw new Exception($"Unexpected {nameof(OperationStatus)} while reading => {internalStatus}"); } + ref Key key = ref pendingContext.key.Get(); + if (pendingContext.heldLatch == LatchOperation.Shared) ReleaseSharedLatch(key); diff --git a/cs/src/core/Index/Interfaces/IFasterKV.cs b/cs/src/core/Index/Interfaces/IFasterKV.cs index b51b3eee1..863502059 100644 --- a/cs/src/core/Index/Interfaces/IFasterKV.cs +++ b/cs/src/core/Index/Interfaces/IFasterKV.cs @@ -206,6 +206,30 @@ IPSF[] RegisterPSF(PSFRegistrationSettings registrationSetting #endregion PSF Registration + #region PSF Logs + // TODO: better interface to PSF logs + + /// + /// Flush PSF logs until current tail (records are still retained in memory) + /// + /// Synchronous wait for operation to complete + void FlushPSFLogs(bool wait); + + /// + /// Flush PSF logs and evict all records from memory + /// + /// Synchronous wait for operation to complete + /// When wait is false, this tells whether the full eviction was successfully registered with FASTER + public bool FlushAndEvictPSFLogs(bool wait); + + /// + /// Delete PSF logs entirely from memory. Cannot allocate on the log + /// after this point. This is a synchronous operation. + /// + public void DisposePSFLogsFromMemory(); + + #endregion PSF Logs + #region Growth and Recovery /// diff --git a/cs/src/core/Index/PSF/DeadRecords.cs b/cs/src/core/Index/PSF/DeadRecords.cs new file mode 100644 index 000000000..b8deddba5 --- /dev/null +++ b/cs/src/core/Index/PSF/DeadRecords.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace FASTER.core.Index.PSF +{ + internal struct DeadRecords + where TRecordId : struct + { + private HashSet deadRecs; + + internal void Add(TRecordId recordId) + { + this.deadRecs ??= new HashSet(); + this.deadRecs.Add(recordId); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool IsDead(TRecordId recordId, bool thisInstanceIsDead) + { + if (thisInstanceIsDead) + { + this.Add(recordId); + return true; + } + return ContainsAndUpdate(recordId, thisInstanceIsDead); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool ContainsAndUpdate(TRecordId recordId, bool thisInstanceIsDead) + { + if (this.deadRecs is null || !this.deadRecs.Contains(recordId)) + return false; + if (!thisInstanceIsDead) // A live record will not be encountered again so remove it + this.Remove(recordId); + return true; + } + + internal void Remove(TRecordId recordId) + { + if (!(this.deadRecs is null)) + this.deadRecs.Remove(recordId); + } + } +} diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs index 4bcbb25a7..b2404cf06 100644 --- a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -3,6 +3,7 @@ using System; using System.Runtime.CompilerServices; +using System.Threading.Tasks; namespace FASTER.core { @@ -27,8 +28,15 @@ internal Status ContextPsfReadKey(ref Key key, ref PSFReadArgs psfAr } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfReadAddress(ref PSFReadArgs psfArgs, long serialNo, - FasterExecutionContext sessionCtx) + internal ValueTask ContextPsfReadKeyAsync(ClientSession clientSession, + ref Key key, ref PSFReadArgs psfArgs, long serialNo, FasterExecutionContext sessionCtx, + PSFQuerySettings querySettings) + { + return PsfReadAsync(clientSession, isKey: true, ref key, ref psfArgs, serialNo, sessionCtx, querySettings); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfReadAddress(ref PSFReadArgs psfArgs, long serialNo, FasterExecutionContext sessionCtx) { var pcontext = default(PendingContext); var internalStatus = this.PsfInternalReadAddress(ref psfArgs, ref pcontext, sessionCtx, serialNo); @@ -40,6 +48,57 @@ internal Status ContextPsfReadAddress(ref PSFReadArgs psfArgs, long return status; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ValueTask ContextPsfReadAddressAsync(ClientSession clientSession, + ref PSFReadArgs psfArgs, long serialNo, FasterExecutionContext sessionCtx, + PSFQuerySettings querySettings) + { + var key = default(Key); + return PsfReadAsync(clientSession, isKey: false, ref key, ref psfArgs, serialNo, sessionCtx, querySettings); + } + + internal ValueTask PsfReadAsync(ClientSession clientSession, bool isKey, + ref Key key, ref PSFReadArgs psfArgs, long serialNo, FasterExecutionContext sessionCtx, + PSFQuerySettings querySettings) + { + var pcontext = default(PendingContext); + var output = default(Output); + var nextSerialNum = clientSession.ctx.serialNum + 1; + + if (clientSession.SupportAsync) clientSession.UnsafeResumeThread(); + try + { + TryReadAgain: + var internalStatus = isKey + ? this.PsfInternalReadKey(ref key, ref psfArgs, ref pcontext, sessionCtx, serialNo) + : this.PsfInternalReadAddress(ref psfArgs, ref pcontext, sessionCtx, serialNo); + if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) + { + return new ValueTask(new ReadAsyncResult((Status)internalStatus, output)); + } + + if (internalStatus == OperationStatus.CPR_SHIFT_DETECTED) + { + SynchronizeEpoch(clientSession.ctx, clientSession.ctx, ref pcontext); + goto TryReadAgain; + } + } + finally + { + clientSession.ctx.serialNum = nextSerialNum; + if (clientSession.SupportAsync) clientSession.UnsafeSuspendThread(); + } + + try + { + return SlowReadAsync(this, clientSession, pcontext, querySettings.CancellationToken); + } + catch (OperationCanceledException) when (!querySettings.ThrowOnCancellation) + { + return default; + } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal Status ContextPsfInsert(ref Key key, ref Value value, ref Input input, long serialNo, FasterExecutionContext sessionCtx) diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index c4d11c394..5095898e4 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -20,23 +20,25 @@ public unsafe partial class FasterKV { - internal IKeyAccessor psfKeyAccessor; + internal IKeyAccessor PsfKeyAccessor => this.hlog.PsfKeyAccessor; + + internal bool ImplmentsPSFs => !(this.PsfKeyAccessor is null); bool ScanQueryChain(ref long logicalAddress, ref Key queryKey, ref int latestRecordVersion) { long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - var recordAddress = this.psfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + var recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); if (latestRecordVersion == -1) latestRecordVersion = hlog.GetInfo(recordAddress).Version; while (true) { - if (this.psfKeyAccessor.Equals(ref queryKey, physicalAddress)) + if (this.PsfKeyAccessor.EqualsAtKeyAddress(ref queryKey, physicalAddress)) { PsfTrace($" / {logicalAddress}"); return true; } - logicalAddress = this.psfKeyAccessor.GetPrevAddress(physicalAddress); + logicalAddress = this.PsfKeyAccessor.GetPrevAddress(physicalAddress); if (logicalAddress < hlog.HeadAddress) break; // RECORD_ON_DISK or not found physicalAddress = hlog.GetPhysicalAddress(logicalAddress); @@ -48,13 +50,13 @@ bool ScanQueryChain(ref long logicalAddress, ref Key queryKey, ref int latestRec [Conditional("PSF_TRACE")] private void PsfTrace(string message) { - if (!(this.psfKeyAccessor is null)) Console.Write(message); + if (!this.ImplmentsPSFs) Console.Write(message); } [Conditional("PSF_TRACE")] private void PsfTraceLine(string message = null) { - if (!(this.psfKeyAccessor is null)) Console.WriteLine(message ?? string.Empty); + if (this.ImplmentsPSFs) Console.WriteLine(message ?? string.Empty); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -71,7 +73,7 @@ internal OperationStatus PsfInternalReadKey( var psfInput = psfArgs.Input; var psfOutput = psfArgs.Output; - var hash = this.psfKeyAccessor.GetHashCode64(ref queryKey, psfInput.PsfOrdinal, isQuery:true); // the queryKey has only one key + var hash = this.PsfKeyAccessor.GetHashCode64(ref queryKey, psfInput.PsfOrdinal, isQuery:true); // the queryKey has only one key var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); if (sessionCtx.phase != Phase.REST) @@ -83,14 +85,14 @@ internal OperationStatus PsfInternalReadKey( OperationStatus status; // For PSFs, the addresses stored in the hash table point to KeyPointer entries, not the record header. - PsfTrace($"ReadKey: {this.psfKeyAccessor?.GetString(ref queryKey, 0)} | hash {hash} |"); + PsfTrace($"ReadKey: {this.PsfKeyAccessor?.GetString(ref queryKey, 0)} | hash {hash} |"); long logicalAddress = Constants.kInvalidAddress; if (tagExists) { logicalAddress = entry.Address; PsfTrace($" {logicalAddress}"); -#if false // TODO: Move from the LogicalAddress to the record header for ReadFromCache +#if false // TODOdcr: Support ReadCache in PSFs (must call this.PsfKeyAccessor.GetRecordAddressFromKeyLogicalAddress) if (UseReadCache && ReadFromCache(ref queryKey, ref logicalAddress, ref physicalAddress, ref latestRecordVersion, psfInput)) { if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) @@ -133,7 +135,7 @@ ref readcache.GetValue(physicalAddress), // Mutable region (even fuzzy region is included here) is above SafeReadOnlyAddress and // is concurrent; Immutable region will not be changed. long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - long recordAddress = this.psfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + long recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); return psfOutput.Visit(psfInput.PsfOrdinal, physicalAddress, ref hlog.GetValue(recordAddress), hlog.GetInfo(recordAddress).Tombstone, @@ -178,8 +180,7 @@ ref hlog.GetValue(recordAddress), pendingContext.output = default; pendingContext.userContext = default; pendingContext.entry.word = entry.word; - pendingContext.logicalAddress = // TODO: fix this in the read callback - this.psfKeyAccessor.GetRecordAddressFromKeyLogicalAddress(logicalAddress, psfInput.PsfOrdinal); + pendingContext.logicalAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyLogicalAddress(logicalAddress, psfInput.PsfOrdinal); pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; pendingContext.heldLatch = heldOperation; @@ -195,7 +196,12 @@ internal OperationStatus PsfInternalReadAddress( ref PSFReadArgs psfArgs, ref PendingContext pendingContext, FasterExecutionContext sessionCtx, long lsn) { - // Note: This function is called for both the primary and secondary FasterKV. + // Notes: + // - This function is called for both the primary and secondary FasterKV. + // - Because we are retrieving a specific address rather than looking up by key, we are not in a position + // to scan for a particular record version--and thus do not consider CPR boundaries, so latestRecordVersion + // is used only as a target for ScanQueryChain. + // TODO: Support a variation of this that allows traversing from a start address -or- the hash table, and returns next start address. var latestRecordVersion = -1; var psfInput = psfArgs.Input; @@ -208,7 +214,7 @@ internal OperationStatus PsfInternalReadAddress( long logicalAddress = psfInput.ReadLogicalAddress; PsfTrace($" ReadAddr: | {logicalAddress}"); -#if false // TODO: Move from the LogicalAddress to the record header for ReadFromCache +#if false // TODOdcr: Support ReadCache in PSFs (must call this.PsfKeyAccessor.GetRecordAddressFromKeyLogicalAddress) if (UseReadCache && ReadFromCache(ref logicalAddress, ref physicalAddress, ref latestRecordVersion)) { if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) @@ -224,25 +230,13 @@ ref readcache.GetValue(physicalAddress), if (logicalAddress >= hlog.HeadAddress) { - if (this.psfKeyAccessor is null) - { - if (latestRecordVersion == -1) - latestRecordVersion = hlog.GetInfo(hlog.GetPhysicalAddress(logicalAddress)).Version; - } - else if (!ScanQueryChain(ref logicalAddress, ref psfInput.QueryKeyRef, ref latestRecordVersion)) + if (this.ImplmentsPSFs && !ScanQueryChain(ref logicalAddress, ref psfInput.QueryKeyRef, ref latestRecordVersion)) { goto ProcessAddress; // RECORD_ON_DISK or not found } } #endregion - if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) - { - PsfTraceLine("CPR_SHIFT_DETECTED"); - status = OperationStatus.CPR_SHIFT_DETECTED; - goto CreatePendingContext; // Pivot thread - } - #region Normal processing ProcessAddress: @@ -252,13 +246,15 @@ ref readcache.GetValue(physicalAddress), // Mutable region (even fuzzy region is included here) is above SafeReadOnlyAddress and // is concurrent; Immutable region will not be changed. long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - if (this.psfKeyAccessor is null) + if (!this.ImplmentsPSFs) + { return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), ref hlog.GetValue(physicalAddress), hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; + } - long recordAddress = this.psfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + long recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); return psfOutput.Visit(psfInput.PsfOrdinal, physicalAddress, ref hlog.GetValue(recordAddress), hlog.GetInfo(recordAddress).Tombstone, @@ -268,10 +264,8 @@ ref hlog.GetValue(recordAddress), // On-Disk Region else if (logicalAddress >= hlog.BeginAddress) { - // We do not have a key here, so we cannot get the hash, latch, etc. for CPR and must retry later; - // this is the only Read operation that goes through Retry rather than pending. TODOtest: Retry - // TODO: Now we have the psfInput.QueryKeyRef so we could get the hashcode; revisit this - status = sessionCtx.phase == Phase.PREPARE ? OperationStatus.RETRY_LATER : OperationStatus.RECORD_ON_DISK; + // As mentioned above, we do not have a key here, so we do not worry about CPR and getting the hash, latching, etc. + status = OperationStatus.RECORD_ON_DISK; goto CreatePendingContext; } else @@ -286,14 +280,14 @@ ref hlog.GetValue(recordAddress), CreatePendingContext: { pendingContext.type = OperationType.PSF_READ_ADDRESS; - pendingContext.key = default; + pendingContext.key = this.ImplmentsPSFs ? hlog.GetKeyContainer(ref psfInput.QueryKeyRef) : default; pendingContext.input = default; pendingContext.output = default; pendingContext.userContext = default; pendingContext.entry.word = default; - pendingContext.logicalAddress = this.psfKeyAccessor is null // TODO: fix this in the read callback - ? logicalAddress - : this.psfKeyAccessor.GetRecordAddressFromKeyLogicalAddress(logicalAddress, psfInput.PsfOrdinal); + pendingContext.logicalAddress = this.ImplmentsPSFs + ? this.PsfKeyAccessor.GetRecordAddressFromKeyLogicalAddress(logicalAddress, psfInput.PsfOrdinal) + : logicalAddress; pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; pendingContext.heldLatch = LatchOperation.None; @@ -327,14 +321,14 @@ internal OperationStatus PsfInternalInsert( // Note: We're not checking for a previous occurrence of the input value (the recordId) because // we are doing insert only here; the update part of upsert is done in PsfInternalUpdate. // TODO: Limit size of stackalloc based on # of PSFs. - var psfCount = this.psfKeyAccessor.KeyCount; + var psfCount = this.PsfKeyAccessor.KeyCount; CASHelper* casHelpers = stackalloc CASHelper[psfCount]; - PsfTrace($"Insert: {this.psfKeyAccessor.GetString(ref compositeKey)} | rId {value} |"); + PsfTrace($"Insert: {this.PsfKeyAccessor.GetString(ref compositeKey)} | rId {value} |"); for (psfInput.PsfOrdinal = 0; psfInput.PsfOrdinal < psfCount; ++psfInput.PsfOrdinal) { // For RCU, or in case we had to retry due to CPR_SHIFT and somehow managed to delete // the previously found record, clear out the chain link pointer. - this.psfKeyAccessor.SetPrevAddress(ref compositeKey, psfInput.PsfOrdinal, Constants.kInvalidAddress); + this.PsfKeyAccessor.SetPrevAddress(ref compositeKey, psfInput.PsfOrdinal, Constants.kInvalidAddress); if (psfInput.IsNullAt) { @@ -343,7 +337,7 @@ internal OperationStatus PsfInternalInsert( } ref CASHelper casHelper = ref casHelpers[psfInput.PsfOrdinal]; - casHelper.hash = this.psfKeyAccessor.GetHashCode64(ref compositeKey, psfInput.PsfOrdinal, isQuery:false); + casHelper.hash = this.PsfKeyAccessor.GetHashCode64(ref compositeKey, psfInput.PsfOrdinal, isQuery:false); var tag = (ushort)((ulong)casHelper.hash >> Constants.kHashTagShift); if (sessionCtx.phase != Phase.REST) @@ -366,19 +360,19 @@ internal OperationStatus PsfInternalInsert( // Note that we do not backtrace here because we are not replacing the value at the key; // instead, we insert at the top of the hash chain. Track the latest record version we've seen. long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - var recordAddress = this.psfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + var recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); if (hlog.GetInfo(physicalAddress).Tombstone) { // The chain might extend past a tombstoned record so we must include it in the chain // unless its prevLink at psfOrdinal is invalid. - var prevAddress = this.psfKeyAccessor.GetPrevAddress(physicalAddress); + var prevAddress = this.PsfKeyAccessor.GetPrevAddress(physicalAddress); if (prevAddress < hlog.BeginAddress) continue; } latestRecordVersion = Math.Max(latestRecordVersion, hlog.GetInfo(recordAddress).Version); } - this.psfKeyAccessor.SetPrevAddress(ref compositeKey, psfInput.PsfOrdinal, logicalAddress); + this.PsfKeyAccessor.SetPrevAddress(ref compositeKey, psfInput.PsfOrdinal, logicalAddress); } else { @@ -404,9 +398,8 @@ internal OperationStatus PsfInternalInsert( #region Create new record in the mutable region CreateNewRecord: { - // Create the new record. Because we are updating multiple hash buckets, mark the - // record as invalid to start, so it is not visible until we have successfully - // updated all chains. + // Create the new record. Because we are updating multiple hash buckets, mark the record as invalid to start, + // so it is not visible until we have successfully updated all chains. var recordSize = hlog.GetRecordSize(ref compositeKey, ref value); BlockAllocate(recordSize, out long newLogicalAddress, sessionCtx); var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); @@ -420,7 +413,7 @@ internal OperationStatus PsfInternalInsert( PsfTraceLine(); newLogicalAddress += RecordInfo.GetLength(); for (psfInput.PsfOrdinal = 0; psfInput.PsfOrdinal < psfCount; - ++psfInput.PsfOrdinal, newLogicalAddress += this.psfKeyAccessor.KeyPointerSize) + ++psfInput.PsfOrdinal, newLogicalAddress += this.PsfKeyAccessor.KeyPointerSize) { var casHelper = casHelpers[psfInput.PsfOrdinal]; var tag = (ushort)((ulong)casHelper.hash >> Constants.kHashTagShift); @@ -453,7 +446,7 @@ internal OperationStatus PsfInternalInsert( { PsfTrace($" / {foundEntry.Address}"); casHelper.entry.word = foundEntry.word; - this.psfKeyAccessor.SetPrevAddress(ref storedKey, psfInput.PsfOrdinal, foundEntry.Address); + this.PsfKeyAccessor.SetPrevAddress(ref storedKey, psfInput.PsfOrdinal, foundEntry.Address); continue; } @@ -500,5 +493,5 @@ internal OperationStatus PsfInternalInsert( ? PsfInternalInsert(ref compositeKey, ref value, ref input, ref pendingContext, sessionCtx, lsn) : status; } - } + } } \ No newline at end of file diff --git a/cs/src/core/Index/PSF/FasterPSFLogOperations.cs b/cs/src/core/Index/PSF/FasterPSFLogOperations.cs new file mode 100644 index 000000000..0723a4172 --- /dev/null +++ b/cs/src/core/Index/PSF/FasterPSFLogOperations.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace FASTER.core +{ + public partial class FasterKV : FasterBase, + IFasterKV + where Key : new() + where Value : new() + where Functions : IFunctions + { + /// + public void FlushPSFLogs(bool wait) => this.PSFManager.FlushLogs(wait); + + /// + public bool FlushAndEvictPSFLogs(bool wait) => this.PSFManager.FlushAndEvictLogs(wait); + + /// + public void DisposePSFLogsFromMemory() => this.PSFManager.DisposeLogsFromMemory(); + } +} diff --git a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs index 449952929..9ceab74d5 100644 --- a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Threading.Tasks; namespace FASTER.core { @@ -42,6 +43,14 @@ internal Status PsfReadKey(ref Key key, ref PSFReadArgs psfArgs, lon } } + internal ValueTask.ReadAsyncResult> PsfReadKeyAsync( + ref Key key, ref PSFReadArgs psfArgs, long serialNo, PSFQuerySettings querySettings) + { + // Called on the secondary FasterKV + return fht.ContextPsfReadKeyAsync(this, ref key, ref psfArgs, serialNo, ctx, querySettings); + } + + internal Status PsfReadAddress(ref PSFReadArgs psfArgs, long serialNo) { // Called on the secondary FasterKV @@ -56,6 +65,13 @@ internal Status PsfReadAddress(ref PSFReadArgs psfArgs, long serialN } } + internal ValueTask.ReadAsyncResult> PsfReadAddressAsync( + ref PSFReadArgs psfArgs, long serialNo, PSFQuerySettings querySettings) + { + // Called on the secondary FasterKV + return fht.ContextPsfReadAddressAsync(this, ref psfArgs, serialNo, ctx, querySettings); + } + internal Status PsfUpdate(ref GroupKeysPair groupKeysPair, ref Value value, ref Input input, long serialNo, PSFChangeTracker changeTracker) { @@ -90,8 +106,7 @@ internal Status PsfDelete(ref Key key, ref Value value, ref Input #region PSF Query API for primary FasterKV - internal Status CreateProviderData(long logicalAddress, - ConcurrentQueue> providerDatas) + internal Status CreateProviderData(long logicalAddress, ConcurrentQueue> providerDatas) { // Looks up logicalAddress in the primary FasterKV var primaryOutput = new PSFOutputPrimaryReadAddress(this.fht.hlog, providerDatas); @@ -101,7 +116,7 @@ internal Status CreateProviderData(long logicalAddress, internal IEnumerable> ReturnProviderDatas(IEnumerable logicalAddresses) { - // If the record is on disk the Read will go pending and we will not receive it "synchronously" + // If the Primary FKV record is on disk the Read will go pending and we will not receive it "synchronously" // here; instead, it will work its way through the pending read system and call psfOutput.Visit. // providerDatas gives that a place to put the record. We should encounter this only after all // non-pending records have been read, but this approach allows any combination of pending and @@ -123,6 +138,36 @@ internal IEnumerable> ReturnProviderDatas(IEnum yield return providerData; } +#if DOTNETCORE + internal async ValueTask> CreateProviderDataAsync(long logicalAddress, ConcurrentQueue> providerDatas, PSFQuerySettings querySettings) + { + // Looks up logicalAddress in the primary FasterKV + var primaryOutput = new PSFOutputPrimaryReadAddress(this.fht.hlog, providerDatas); + var psfArgs = new PSFReadArgs(new PSFInputPrimaryReadAddress(logicalAddress), primaryOutput); + var readAsyncResult = await this.PsfReadAddressAsync(ref psfArgs, this.ctx.serialNum + 1, querySettings); + if (querySettings.IsCanceled) + return null; + var (status, _) = readAsyncResult.CompleteRead(); + if (status != Status.OK) // TODOerr: check other status + return null; + return providerDatas.TryDequeue(out var providerData) ? providerData : null; + } + + internal async IAsyncEnumerable> ReturnProviderDatasAsync(IAsyncEnumerable logicalAddresses, PSFQuerySettings querySettings) + { + querySettings ??= PSFQuerySettings.Default; + + // For the async form, we always read fully; there is no pending. + var providerDatas = new ConcurrentQueue>(); + await foreach (var logicalAddress in logicalAddresses) + { + var providerData = await this.CreateProviderDataAsync(logicalAddress, providerDatas, querySettings); + if (!(providerData is null)) + yield return providerData; + } + } +#endif + /// /// Issue a query on a single on a single key value. /// @@ -143,6 +188,28 @@ public IEnumerable> QueryPSF( return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, key, querySettings)); } +#if DOTNETCORE + /// + /// Issue a query on a single on a single key value. + /// + /// + /// foreach (var providerData in fht.QueryPSF(sizePsf, Size.Medium)) {...} + /// + /// The type of the key value to return results for + /// The Predicate Subset Function object + /// The key value to return results for + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf, TPSFKey key, PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf, key, querySettings), querySettings); + } +#endif // DOTNETCORE + /// /// Issue a query on a single on multiple key values. /// @@ -165,6 +232,30 @@ public IEnumerable> QueryPSF( return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, keys, querySettings)); } +#if DOTNETCORE + /// + /// Issue a query on a single on multiple key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF(sizePsf, new TestPSFKey[] { Size.Medium, Size.Large })) {...} + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). + /// + /// The type of the key value to return results for + /// The Predicate Subset Function object + /// A vector of key values to return results for; for example, an OR query on + /// a single PSF, or a range query for a PSF that generates keys identifying bins. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf, IEnumerable keys, PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf, keys, querySettings), querySettings); + } +#endif // DOTNETCORE + /// /// Issue a query on two s, each with a single key value. /// @@ -196,6 +287,39 @@ public IEnumerable> QueryPSF + /// Issue a query on two s, each with a single key value. + /// + /// + /// var providerData in fht.QueryPSF(sizePsf, Size.Medium, colorPsf, Color.Red, (l, r) => l || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The key value to return results from the first 's stored values + /// The key value to return results from the second 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf1, key1, psf2, key2, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + /// /// Issue a query on two s, each with a vector of key values. /// @@ -230,6 +354,42 @@ public IEnumerable> QueryPSF + /// Issue a query on two s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }, + /// colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }, + /// (l, r) => l || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The first Predicate Subset Function object + /// The secojnd Predicate Subset Function object + /// The key values to return results from the first 's stored values + /// The key values to return results from the second 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf1, keys1, psf2, keys2, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + /// /// Issue a query on three s, each with a single key value. /// @@ -266,6 +426,44 @@ public IEnumerable> QueryPSF + /// Issue a query on three s, each with a single key value. + /// + /// + /// var providerData in fht.QueryPSF(sizePsf, Size.Medium, colorPsf, Color.Red, countPsf, 7, (l, m, r) => l || m || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The type of the key value for the third + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The third Predicate Subset Function object + /// The key value to return results from the first 's stored values + /// The key value to return results from the second 's stored values + /// The key value to return results from the third 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, 3) whether the record matches the third PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + IPSF psf3, TPSFKey3 key3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf1, key1, psf2, key2, psf3, key3, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + /// /// Issue a query on three s, each with a vector of key values. /// @@ -306,6 +504,48 @@ public IEnumerable> QueryPSF + /// Issue a query on three s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }, + /// colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }, + /// countPsf, new [] { new CountKey(7), new CountKey(42) }, + /// (l, m, r) => l || m || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The type of the key value for the third + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The third Predicate Subset Function object + /// The key values to return results from the first 's stored values + /// The key values to return results from the second 's stored values + /// The key values to return results from the third 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, 3) whether the record matches the third PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + IPSF psf3, IEnumerable keys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf1, keys1, psf2, keys2, psf3, keys3, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + /// /// Issue a query on one or more s, each with a vector of key values. /// @@ -337,6 +577,39 @@ public IEnumerable> QueryPSF( return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psfsAndKeys, matchPredicate, querySettings)); } +#if DOTNETCORE + /// + /// Issue a query on one or more s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { + /// (sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), + /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue})}, + /// ll => ll[0])) + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). + /// + /// The type of the key value for the vector + /// A vector of s and associated keys to be queried + /// A predicate that takes as a parameters a boolean vector in parallel with + /// the vector indicating whether a candidate record matches the corresponding + /// , and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of the input vector are true, + /// else false; an OR query would return true if element of the input vector is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psfsAndKeys, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + /// /// Issue a query on multiple keys s for two different key types. /// @@ -377,6 +650,48 @@ public IEnumerable> QueryPSF + /// Issue a query on multiple keys s for two different key types. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { + /// (sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), + /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue })}, + /// new[] { + /// (countPsf, new [] { new CountKey(7), new CountKey(9) })}, + /// (ll, rr) => ll[0] || rr[0])) + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). + /// + /// The type of the key value for the first vector's s + /// The type of the key value for the second vector's s + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A predicate that takes as a parameters a boolean vector in parallel with + /// the vector and a second boolean vector in parallel with + /// the vector, and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of both input vectors are true, + /// else false; an OR query would return true if any element of either input vector is true; and more complex + /// logic could be done depending on the specific PSFs. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psfsAndKeys1, psfsAndKeys2, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + /// /// Issue a query on multiple keys s for three different key types. /// @@ -417,6 +732,50 @@ public IEnumerable> QueryPSF + /// Issue a query on multiple keys s for three different key types. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { (sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }) }, + /// new[] { (colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }) }, + /// new[] { (countPsf, new [] { new CountKey(4), new CountKey(7) }) }, + /// (ll, mm, rr) => ll[0] || mm[0] || rr[0])) + /// + /// The type of the key value for the first vector's s + /// The type of the key value for the second vector's s + /// The type of the key value for the third vector's s + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A predicate that takes as a parameters three boolean vectors in parallel with + /// each other, and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of all input vectors are true, + /// else false; an OR query would return true if any element of either input vector is true; and more complex + /// logic could be done depending on the specific PSFs. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psfsAndKeys1, psfsAndKeys2, psfsAndKeys3, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + #endregion PSF Query API for primary FasterKV } } diff --git a/cs/src/core/Index/PSF/IExecutePSF.cs b/cs/src/core/Index/PSF/IExecutePSF.cs index e17d01c8c..fd5ac59a0 100644 --- a/cs/src/core/Index/PSF/IExecutePSF.cs +++ b/cs/src/core/Index/PSF/IExecutePSF.cs @@ -76,5 +76,24 @@ Status ExecuteAndStore(TProviderData data, TRecordId recordId, PSFExecutePhase p /// Recover from last successful checkpoints /// void Recover(); + + /// + /// Flush PSF logs until current tail (records are still retained in memory) + /// + /// Synchronous wait for operation to complete + void FlushLog(bool wait); + + /// + /// Flush PSF logs and evict all records from memory + /// + /// Synchronous wait for operation to complete + /// When wait is false, this tells whether the full eviction was successfully registered with FASTER + public bool FlushAndEvictLog(bool wait); + + /// + /// Delete PSF logs entirely from memory. Cannot allocate on the log + /// after this point. This is a synchronous operation. + /// + public void DisposeLogFromMemory(); } } diff --git a/cs/src/core/Index/PSF/IQueryPSF.cs b/cs/src/core/Index/PSF/IQueryPSF.cs index 1ec231af9..52ae6bdf0 100644 --- a/cs/src/core/Index/PSF/IQueryPSF.cs +++ b/cs/src/core/Index/PSF/IQueryPSF.cs @@ -18,7 +18,19 @@ public interface IQueryPSF /// /// The ordinal of the in this group /// The key to query on to rertrieve the s. + /// Optional query settings for EOS, cancellation, etc. /// - IEnumerable Query(int psfOrdinal, TPSFKey key); + IEnumerable Query(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings); + +#if DOTNETCORE + /// + /// Issues a query on the specified to return s. + /// + /// The ordinal of the in this group + /// The key to query on to rertrieve the s. + /// Optional query settings for EOS, cancellation, etc. + /// + IAsyncEnumerable QueryAsync(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings); +#endif } } diff --git a/cs/src/core/Index/PSF/KeyAccessor.cs b/cs/src/core/Index/PSF/KeyAccessor.cs index 631471d93..ca10a4876 100644 --- a/cs/src/core/Index/PSF/KeyAccessor.cs +++ b/cs/src/core/Index/PSF/KeyAccessor.cs @@ -22,11 +22,15 @@ internal interface IKeyAccessor long GetRecordAddressFromKeyPhysicalAddress(long physicalAddress); - long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, long psfOrdinal); + long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, int psfOrdinal); + + long GetKeyAddressFromRecordPhysicalAddress(long physicalAddress, int psfOrdinal); long GetHashCode64(ref TCompositeKey key, int psfOrdinal, bool isQuery); - bool Equals(ref TCompositeKey queryKey, long physicalAddress); + bool EqualsAtKeyAddress(ref TCompositeKey queryKey, long physicalAddress); + + bool EqualsAtRecordAddress(ref TCompositeKey queryKey, long physicalAddress); string GetString(ref TCompositeKey key, int psfOrdinal = -1); } @@ -62,25 +66,44 @@ public void SetPrevAddress(ref CompositeKey key, int psfOrdinal, long p [MethodImpl(MethodImplOptions.AggressiveInlining)] public long GetRecordAddressFromKeyPhysicalAddress(long physicalAddress) - => physicalAddress - this.GetKeyPointerRef(physicalAddress).PsfOrdinal * this.KeyPointerSize - RecordInfo.GetLength(); + => physicalAddress - this.GetKeyPointerRef(physicalAddress).PsfOrdinal * this.KeyPointerSize - RecordInfo.GetLength(); // TODO: Assumes all PSFs are present + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, int psfOrdinal) + => logicalAddress - psfOrdinal * this.KeyPointerSize - RecordInfo.GetLength(); // TODO: Assumes all PSFs are present [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, long psfOrdinal) - => logicalAddress - psfOrdinal* this.KeyPointerSize - RecordInfo.GetLength(); // TODO: Assumes all PSFs are present + public long GetKeyAddressFromRecordPhysicalAddress(long physicalAddress, int psfOrdinal) + => physicalAddress + RecordInfo.GetLength() + psfOrdinal * this.KeyPointerSize; // TODO: Assumes all PSFs are present [MethodImpl(MethodImplOptions.AggressiveInlining)] public long GetHashCode64(ref CompositeKey key, int psfOrdinal, bool isQuery) { - ref KeyPointer queryKeyPointer = ref key.GetKeyPointerRef(isQuery ? 0 : psfOrdinal, this.KeyPointerSize); - return Utility.GetHashCode(this.userComparer.GetHashCode64(ref queryKeyPointer.Key)) ^ Utility.GetHashCode(psfOrdinal + 1); + ref KeyPointer keyPointer = ref key.GetKeyPointerRef(isQuery ? 0 : psfOrdinal, this.KeyPointerSize); + return Utility.GetHashCode(this.userComparer.GetHashCode64(ref keyPointer.Key)) ^ Utility.GetHashCode(keyPointer.PsfOrdinal + 1); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool Equals(ref CompositeKey queryKey, long physicalAddress) + public bool EqualsAtKeyAddress(ref CompositeKey queryKey, long physicalAddress) { // The query key only has a single key in it--the one we're trying to match. ref KeyPointer queryKeyPointer = ref queryKey.GetKeyPointerRef(0, this.KeyPointerSize); ref KeyPointer storedKeyPointer = ref GetKeyPointerRef(physicalAddress); + return KeysEqual(ref queryKeyPointer, ref storedKeyPointer); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool EqualsAtRecordAddress(ref CompositeKey queryKey, long physicalAddress) + { + // The query key only has a single key in it--the one we're trying to match. + ref KeyPointer queryKeyPointer = ref queryKey.GetKeyPointerRef(0, this.KeyPointerSize); + ref KeyPointer storedKeyPointer = ref GetKeyPointerRef(physicalAddress, queryKeyPointer.PsfOrdinal); + return KeysEqual(ref queryKeyPointer, ref storedKeyPointer); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool KeysEqual(ref KeyPointer queryKeyPointer, ref KeyPointer storedKeyPointer) + { return queryKeyPointer.PsfOrdinal == storedKeyPointer.PsfOrdinal && this.userComparer.Equals(ref queryKeyPointer.Key, ref storedKeyPointer.Key); } @@ -93,6 +116,10 @@ internal ref KeyPointer GetKeyPointerRef(ref CompositeKey key, internal unsafe ref KeyPointer GetKeyPointerRef(long physicalAddress) => ref Unsafe.AsRef>((byte*)physicalAddress); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal unsafe ref KeyPointer GetKeyPointerRef(long physicalAddress, int psfOrdinal) + => ref Unsafe.AsRef>((byte*)GetKeyAddressFromRecordPhysicalAddress(physicalAddress, psfOrdinal)); + public string GetString(ref CompositeKey key, int psfOrdinal = -1) { var sb = new StringBuilder("{"); diff --git a/cs/src/core/Index/PSF/PSF.cs b/cs/src/core/Index/PSF/PSF.cs index 34951a299..9cab5d535 100644 --- a/cs/src/core/Index/PSF/PSF.cs +++ b/cs/src/core/Index/PSF/PSF.cs @@ -43,6 +43,15 @@ internal PSF(long groupId, int psfOrdinal, string name, IQueryPSF /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal IEnumerable Query(TPSFKey key) => this.psfGroup.Query(this.PsfOrdinal, key); + internal IEnumerable Query(TPSFKey key, PSFQuerySettings querySettings) => this.psfGroup.Query(this.PsfOrdinal, key, querySettings); + +#if DOTNETCORE + /// + /// Issues a query on this PSF to return s. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal IAsyncEnumerable QueryAsync(TPSFKey key, PSFQuerySettings querySettings) => this.psfGroup.QueryAsync(this.PsfOrdinal, key, querySettings); +#endif } } diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index 43cd99491..c3a43d051 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. +using FASTER.core.Index.PSF; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -103,7 +104,8 @@ public PSFGroup(PSFRegistrationSettings regSettings, IPSFDefinition.VarLenLength(this.keyPointerSize, this.PSFCount) } - ) { psfKeyAccessor = this.keyAccessor}; + ); + this.fht.hlog.PsfKeyAccessor = keyAccessor; this.bufferPool = this.fht.hlog.bufferPool; } @@ -328,20 +330,20 @@ public Status Delete(PSFChangeTracker changeTracker) } /// - public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key) + public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings) { // Putting the query key in PSFInput is necessary because iterator functions cannot contain unsafe code or have // byref args, and bufferPool is needed here because the stack goes away as part of the iterator operation. // TODOperf: PSFInput* and PSFOutput* are classes because we communicate through interfaces to avoid // having to add additional generic args. Interfaces on structs incur boxing overhead (plus the complexity // of mutable structs). But check the performance here; if necessary perhaps I can figure out a way to - // pass a struct with no TPSFKey, TRecordId, etc. and use an FHT-level interface to manage it. + // pass a struct with no TPSFKey, TRecordId, etc. and use an FHT-level interface to manage it (async too). var psfInput = new PSFInputSecondary(psfOrdinal, this.keyAccessor, this.Id); psfInput.SetQueryKey(this.bufferPool, ref key); - return Query(psfInput); + return Query(psfInput, querySettings); } - private IEnumerable Query(PSFInputSecondary input) + private IEnumerable Query(PSFInputSecondary input, PSFQuerySettings querySettings) { // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr. Also consider @@ -350,26 +352,23 @@ private IEnumerable Query(PSFInputSecondary input) var readArgs = new PSFReadArgs, TRecordId>(input, secondaryOutput); var session = this.GetSession(); - HashSet deadRecs = null; + var deadRecs = new DeadRecords(); try { // Because we traverse the chain, we must wait for any pending read operations to complete. // TODOperf: See if there is a better solution than spinWaiting in CompletePending. Status status = session.PsfReadKey(ref input.QueryKeyRef, ref readArgs, session.ctx.serialNum + 1); + if (querySettings.IsCanceled) + yield break; if (status == Status.PENDING) session.CompletePending(spinWait: true); if (status != Status.OK) // TODOerr: check other status yield break; if (secondaryOutput.Tombstone) - { - deadRecs ??= new HashSet(); deadRecs.Add(secondaryOutput.RecordId); - } else - { yield return secondaryOutput.RecordId; - } do { @@ -377,22 +376,74 @@ private IEnumerable Query(PSFInputSecondary input) status = session.PsfReadAddress(ref readArgs, session.ctx.serialNum + 1); if (status == Status.PENDING) session.CompletePending(spinWait: true); + if (querySettings.IsCanceled) + yield break; if (status != Status.OK) // TODOerr: check other status yield break; - - if (secondaryOutput.Tombstone) - { - deadRecs ??= new HashSet(); - deadRecs.Add(secondaryOutput.RecordId); + + if (deadRecs.IsDead(secondaryOutput.RecordId, secondaryOutput.Tombstone)) continue; - } - if (!(deadRecs is null) && deadRecs.Contains(secondaryOutput.RecordId)) - { - if (!secondaryOutput.Tombstone) // A live record will not be encountered again so remove it - deadRecs.Remove(secondaryOutput.RecordId); + yield return secondaryOutput.RecordId; + } while (secondaryOutput.PreviousLogicalAddress != Constants.kInvalidAddress); + } + finally + { + this.ReleaseSession(session); + input.Dispose(); + } + } + +#if DOTNETCORE + /// + public unsafe IAsyncEnumerable QueryAsync(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings) + { + // See comments on non-async version. + var psfInput = new PSFInputSecondary(psfOrdinal, this.keyAccessor, this.Id); + psfInput.SetQueryKey(this.bufferPool, ref key); + return QueryAsync(psfInput, querySettings); + } + + private async IAsyncEnumerable QueryAsync(PSFInputSecondary input, PSFQuerySettings querySettings) + { + // TODO: Improved enumerator in DurTask + + // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them + // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr. Also consider + // having TPSFKeys[] for a single PSF walk through in parallel, so the FHT log memory access is sequential. + var secondaryOutput = new PSFOutputSecondary(this.keyAccessor); + var readArgs = new PSFReadArgs, TRecordId>(input, secondaryOutput); + + var session = this.GetSession(); + var deadRecs = new DeadRecords(); + try + { + // Because we traverse the chain, we must wait for any pending read operations to complete. + var readAsyncResult = await session.PsfReadKeyAsync(ref input.QueryKeyRef, ref readArgs, session.ctx.serialNum + 1, querySettings); + if (querySettings.IsCanceled) + yield break; + var (status, _) = readAsyncResult.CompleteRead(); + if (status != Status.OK) // TODOerr: check other status + yield break; + + if (secondaryOutput.Tombstone) + deadRecs.Add(secondaryOutput.RecordId); + else + yield return secondaryOutput.RecordId; + + do + { + readArgs.Input.ReadLogicalAddress = secondaryOutput.PreviousLogicalAddress; + readAsyncResult = await session.PsfReadAddressAsync(ref readArgs, session.ctx.serialNum + 1, querySettings); + if (querySettings.IsCanceled) + yield break; + (status, _) = readAsyncResult.CompleteRead(); + if (status != Status.OK) // TODOerr: check other status + yield break; + + if (deadRecs.IsDead(secondaryOutput.RecordId, secondaryOutput.Tombstone)) continue; - } + yield return secondaryOutput.RecordId; } while (secondaryOutput.PreviousLogicalAddress != Constants.kInvalidAddress); } @@ -402,7 +453,9 @@ private IEnumerable Query(PSFInputSecondary input) input.Dispose(); } } +#endif + #region Checkpoint Operations /// public bool TakeFullCheckpoint() => this.fht.TakeFullCheckpoint(out _); @@ -417,5 +470,17 @@ private IEnumerable Query(PSFInputSecondary input) /// public void Recover() => this.fht.Recover(); + #endregion Checkpoint Operations + + #region Log Operations + /// + public void FlushLog(bool wait) => this.fht.Log.Flush(wait); + + /// + public bool FlushAndEvictLog(bool wait) => this.fht.Log.FlushAndEvict(wait); + + /// + public void DisposeLogFromMemory() => this.fht.Log.DisposeFromMemory(); + #endregion Log Operations } } diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index 36f3c2314..366441726 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -20,9 +20,6 @@ private readonly ConcurrentDictionary psfNames = new ConcurrentDictionary(); - // Default is to let all streams continue to completion. - private static readonly PSFQuerySettings DefaultQuerySettings = new PSFQuerySettings { OnStreamEnded = (unusedPsf, unusedIndex) => true }; - internal bool HasPSFs => this.psfGroups.Count > 0; internal Status Upsert(TProviderData data, TRecordId recordId, PSFChangeTracker changeTracker) @@ -129,12 +126,15 @@ private PSF GetImplementingPSF(IPSF ipsf) return psf; } - private void VerifyIsOurPSF(IPSF psf) + private void VerifyIsOurPSF(params IPSF[] psfs) { - if (psf is null) - throw new PSFArgumentException($"The PSF cannot be null."); - if (!this.psfNames.ContainsKey(psf.Name)) - throw new PSFArgumentException($"The PSF {psf.Name} is not registered with this FasterKV."); + foreach (var psf in psfs) + { + if (psf is null) + throw new PSFArgumentException($"The PSF cannot be null."); + if (!this.psfNames.ContainsKey(psf.Name)) + throw new PSFArgumentException($"The PSF {psf.Name} is not registered with this FasterKV."); + } } private void VerifyIsOurPSF(IEnumerable<(IPSF, IEnumerable)> psfsAndKeys) @@ -142,7 +142,23 @@ private void VerifyIsOurPSF(IEnumerable<(IPSF, IEnumerable)> p if (psfsAndKeys is null) throw new PSFArgumentException($"The PSF enumerable cannot be null."); foreach (var psfAndKeys in psfsAndKeys) - this.VerifyIsOurPSF(psfAndKeys.Item1); + this.VerifyIsOurPSF(psfAndKeys.Item1); + } + + private void VerifyIsOurPSF(IEnumerable<(IPSF, IEnumerable)> psfsAndKeys1, + IEnumerable<(IPSF, IEnumerable)> psfsAndKeys2) + { + VerifyIsOurPSF(psfsAndKeys1); + VerifyIsOurPSF(psfsAndKeys2); + } + + private void VerifyIsOurPSF(IEnumerable<(IPSF, IEnumerable)> psfsAndKeys1, + IEnumerable<(IPSF, IEnumerable)> psfsAndKeys2, + IEnumerable<(IPSF, IEnumerable)> psfsAndKeys3) + { + VerifyIsOurPSF(psfsAndKeys1); + VerifyIsOurPSF(psfsAndKeys2); + VerifyIsOurPSF(psfsAndKeys3); } private static void VerifyRegistrationSettings(PSFRegistrationSettings registrationSettings) where TPSFKey : struct @@ -153,6 +169,10 @@ private static void VerifyRegistrationSettings(PSFRegistrationSettings< throw new PSFArgumentException("PSFRegistrationSettings.LogSettings is required"); if (registrationSettings.CheckpointSettings is null) throw new PSFArgumentException("PSFRegistrationSettings.CheckpointSettings is required"); + + // TODOdcr: Support ReadCache and CopyReadsToTail for PSFs + if (!(registrationSettings.LogSettings.ReadCacheSettings is null) || registrationSettings.LogSettings.CopyReadsToTail) + throw new PSFArgumentException("PSFs do not support ReadCache or CopyReadsToTail"); } internal IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, IPSFDefinition def) @@ -213,8 +233,8 @@ internal IEnumerable QueryPSF(IPSF psf, TPSFKey key, PSFQuer where TPSFKey : struct { var psfImpl = this.GetImplementingPSF(psf); - querySettings ??= DefaultQuerySettings; - foreach (var recordId in psfImpl.Query(key)) + querySettings ??= PSFQuerySettings.Default; + foreach (var recordId in psfImpl.Query(key, querySettings)) { if (querySettings.IsCanceled) yield break; @@ -222,11 +242,27 @@ internal IEnumerable QueryPSF(IPSF psf, TPSFKey key, PSFQuer } } +#if DOTNETCORE + internal async IAsyncEnumerable QueryPSFAsync(IPSF psf, TPSFKey key, PSFQuerySettings querySettings) + where TPSFKey : struct + { + var psfImpl = this.GetImplementingPSF(psf); + querySettings ??= PSFQuerySettings.Default; + await foreach (var recordId in psfImpl.QueryAsync(key, querySettings)) + { + if (querySettings.IsCanceled) + yield break; + yield return recordId; + } + } + +#endif // DOTNETCORE + internal IEnumerable QueryPSF(IPSF psf, IEnumerable keys, PSFQuerySettings querySettings) where TPSFKey : struct { - this.VerifyIsOurPSF(psf); - querySettings ??= DefaultQuerySettings; + this.VerifyIsOurPSF(psf); + querySettings ??= PSFQuerySettings.Default; // The recordIds cannot overlap between keys (unless something's gone wrong), so return them all. // TODOperf: Consider a PQ ordered on secondary FKV LA so we can walk through in parallel (and in memory sequence) in one PsfRead(Key|Address) loop. @@ -241,6 +277,28 @@ internal IEnumerable QueryPSF(IPSF psf, IEnumerable } } +#if DOTNETCORE + internal async IAsyncEnumerable QueryPSFAsync(IPSF psf, IEnumerable keys, PSFQuerySettings querySettings) + where TPSFKey : struct + { + this.VerifyIsOurPSF(psf); + querySettings ??= PSFQuerySettings.Default; + + // The recordIds cannot overlap between keys (unless something's gone wrong), so return them all. + // TODOperf: Consider a PQ ordered on secondary FKV LA so we can walk through in parallel (and in memory sequence) in one PsfRead(Key|Address) loop. + foreach (var key in keys) + { + await foreach (var recordId in QueryPSFAsync(psf, key, querySettings)) + { + if (querySettings.IsCanceled) + yield break; + yield return recordId; + } + } + } + +#endif // DOTNETCORE + internal IEnumerable QueryPSF( IPSF psf1, TPSFKey1 key1, IPSF psf2, TPSFKey2 key2, @@ -249,14 +307,31 @@ internal IEnumerable QueryPSF( where TPSFKey1 : struct where TPSFKey2 : struct { - this.VerifyIsOurPSF(psf1); - this.VerifyIsOurPSF(psf2); - querySettings ??= DefaultQuerySettings; + this.VerifyIsOurPSF(psf1, psf2); + querySettings ??= PSFQuerySettings.Default; return new QueryRecordIterator(psf1, this.QueryPSF(psf1, key1, querySettings), psf2, this.QueryPSF(psf2, key2, querySettings), matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0]), querySettings).Run(); } +#if DOTNETCORE + internal IAsyncEnumerable QueryPSFAsync( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + Func matchPredicate, + PSFQuerySettings querySettings) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psf1, psf2); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(psf1, this.QueryPSFAsync(psf1, key1, querySettings), psf2, this.QueryPSFAsync(psf2, key2, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0]), querySettings).Run(); + } + +#endif // DOTNETCORE + internal IEnumerable QueryPSF( IPSF psf1, IEnumerable keys1, IPSF psf2, IEnumerable keys2, @@ -265,14 +340,30 @@ internal IEnumerable QueryPSF( where TPSFKey1 : struct where TPSFKey2 : struct { - this.VerifyIsOurPSF(psf1); - this.VerifyIsOurPSF(psf2); - querySettings ??= DefaultQuerySettings; + this.VerifyIsOurPSF(psf1, psf2); + querySettings ??= PSFQuerySettings.Default; return new QueryRecordIterator(psf1, this.QueryPSF(psf1, keys1, querySettings), psf2, this.QueryPSF(psf2, keys2, querySettings), matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0]), querySettings).Run(); } +#if DOTNETCORE + internal IAsyncEnumerable QueryPSFAsync( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + Func matchPredicate, + PSFQuerySettings querySettings) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psf1, psf2); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(psf1, this.QueryPSFAsync(psf1, keys1, querySettings), psf2, this.QueryPSFAsync(psf2, keys2, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0]), querySettings).Run(); + } +#endif // DOTNETCORE + public IEnumerable QueryPSF( IPSF psf1, TPSFKey1 key1, IPSF psf2, TPSFKey2 key2, @@ -283,16 +374,34 @@ public IEnumerable QueryPSF( where TPSFKey2 : struct where TPSFKey3 : struct { - this.VerifyIsOurPSF(psf1); - this.VerifyIsOurPSF(psf2); - this.VerifyIsOurPSF(psf3); - querySettings ??= DefaultQuerySettings; + this.VerifyIsOurPSF(psf1, psf2, psf3); + querySettings ??= PSFQuerySettings.Default; return new QueryRecordIterator(psf1, this.QueryPSF(psf1, key1, querySettings), psf2, this.QueryPSF(psf2, key2, querySettings), psf3, this.QueryPSF(psf3, key3, querySettings), matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0], matchIndicators[2][0]), querySettings).Run(); } +#if DOTNETCORE + public IAsyncEnumerable QueryPSFAsync( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + IPSF psf3, TPSFKey3 key3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + this.VerifyIsOurPSF(psf1, psf2, psf3); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(psf1, this.QueryPSFAsync(psf1, key1, querySettings), psf2, this.QueryPSFAsync(psf2, key2, querySettings), + psf3, this.QueryPSFAsync(psf3, key3, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0], matchIndicators[2][0]), querySettings).Run(); + } +#endif // DOTNETCORE + public IEnumerable QueryPSF( IPSF psf1, IEnumerable keys1, IPSF psf2, IEnumerable keys2, @@ -303,17 +412,35 @@ public IEnumerable QueryPSF( where TPSFKey2 : struct where TPSFKey3 : struct { - this.VerifyIsOurPSF(psf1); - this.VerifyIsOurPSF(psf2); - this.VerifyIsOurPSF(psf3); - querySettings ??= DefaultQuerySettings; + this.VerifyIsOurPSF(psf1, psf2, psf3); + querySettings ??= PSFQuerySettings.Default; return new QueryRecordIterator(psf1, this.QueryPSF(psf1, keys1, querySettings), psf2, this.QueryPSF(psf2, keys2, querySettings), psf3, this.QueryPSF(psf3, keys3, querySettings), matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0], matchIndicators[2][0]), querySettings).Run(); } - // Power user versions. Anything more complicated than this the caller can post-process with LINQ. +#if DOTNETCORE + public IAsyncEnumerable QueryPSFAsync( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + IPSF psf3, IEnumerable keys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + this.VerifyIsOurPSF(psf1, psf2, psf3); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(psf1, this.QueryPSFAsync(psf1, keys1, querySettings), psf2, this.QueryPSFAsync(psf2, keys2, querySettings), + psf3, this.QueryPSFAsync(psf3, keys3, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0], matchIndicators[2][0]), querySettings).Run(); + } +#endif // DOTNETCORE + + // Power user versions. Anything more complicated than this can be post-processed with LINQ. internal IEnumerable QueryPSF( IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, @@ -322,12 +449,27 @@ internal IEnumerable QueryPSF( where TPSFKey : struct { this.VerifyIsOurPSF(psfsAndKeys); - querySettings ??= DefaultQuerySettings; + querySettings ??= PSFQuerySettings.Default; return new QueryRecordIterator(new[] { psfsAndKeys.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))) }, matchIndicators => matchPredicate(matchIndicators[0]), querySettings).Run(); } +#if DOTNETCORE + internal IAsyncEnumerable QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + this.VerifyIsOurPSF(psfsAndKeys); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(new[] { psfsAndKeys.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings))) }, + matchIndicators => matchPredicate(matchIndicators[0]), querySettings).Run(); + } +#endif // DOTNETCORE + internal IEnumerable QueryPSF( IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, @@ -336,15 +478,32 @@ internal IEnumerable QueryPSF( where TPSFKey1 : struct where TPSFKey2 : struct { - this.VerifyIsOurPSF(psfsAndKeys1); - this.VerifyIsOurPSF(psfsAndKeys2); - querySettings ??= DefaultQuerySettings; + this.VerifyIsOurPSF(psfsAndKeys1, psfsAndKeys2); + querySettings ??= PSFQuerySettings.Default; return new QueryRecordIterator(new[] {psfsAndKeys1.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))), psfsAndKeys2.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings)))}, matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1]), querySettings).Run(); } +#if DOTNETCORE + internal IAsyncEnumerable QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psfsAndKeys1, psfsAndKeys2); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(new[] {psfsAndKeys1.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings))), + psfsAndKeys2.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings)))}, + matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1]), querySettings).Run(); + } +#endif // DOTNETCORE + internal IEnumerable QueryPSF( IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, @@ -355,10 +514,8 @@ internal IEnumerable QueryPSF( where TPSFKey2 : struct where TPSFKey3 : struct { - this.VerifyIsOurPSF(psfsAndKeys1); - this.VerifyIsOurPSF(psfsAndKeys2); - this.VerifyIsOurPSF(psfsAndKeys3); - querySettings ??= DefaultQuerySettings; + this.VerifyIsOurPSF(psfsAndKeys1, psfsAndKeys2, psfsAndKeys3); + querySettings ??= PSFQuerySettings.Default; return new QueryRecordIterator(new[] {psfsAndKeys1.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))), psfsAndKeys2.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))), @@ -366,6 +523,28 @@ internal IEnumerable QueryPSF( matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1], matchIndicators[2]), querySettings).Run(); } +#if DOTNETCORE + internal IAsyncEnumerable QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + this.VerifyIsOurPSF(psfsAndKeys1, psfsAndKeys2, psfsAndKeys3); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(new[] {psfsAndKeys1.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings))), + psfsAndKeys2.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings))), + psfsAndKeys3.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings)))}, + matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1], matchIndicators[2]), querySettings).Run(); + } +#endif // DOTNETCORE + + #region Checkpoint Operations // TODO Separate Tasks for each group's commit/restore operations? public bool TakeFullCheckpoint() => this.psfGroups.Values.Aggregate(true, (result, group) => group.TakeFullCheckpoint() && result); @@ -387,5 +566,42 @@ public void Recover() foreach (var group in this.psfGroups.Values) group.Recover(); } + #endregion Checkpoint Operations + + #region Log Operations + + public void FlushLogs(bool wait) + { + foreach (var group in this.psfGroups.Values) + group.FlushLog(wait); + } + + /// + /// Flush log and evict all records from memory + /// + /// Synchronous wait for operation to complete + /// When wait is false, this tells whether the full eviction was successfully registered with FASTER + public bool FlushAndEvictLogs(bool wait) + { + foreach (var group in this.psfGroups.Values) + { + if (!group.FlushAndEvictLog(wait)) + { + // TODO handle error on FlushAndEvictLogs + } + } + return true; + } + + /// + /// Delete log entirely from memory. Cannot allocate on the log + /// after this point. This is a synchronous operation. + /// + public void DisposeLogsFromMemory() + { + foreach (var group in this.psfGroups.Values) + group.DisposeLogFromMemory(); + } + #endregion Log Operations } } diff --git a/cs/src/core/Index/PSF/PSFQuerySettings.cs b/cs/src/core/Index/PSF/PSFQuerySettings.cs index 5fd25575d..f45ca64d1 100644 --- a/cs/src/core/Index/PSF/PSFQuerySettings.cs +++ b/cs/src/core/Index/PSF/PSFQuerySettings.cs @@ -41,5 +41,8 @@ internal bool IsCanceled } internal bool CancelOnEOS(IPSF psf, (int, int) location) => !(this.OnStreamEnded is null) && !this.OnStreamEnded(psf, location); + + // Default is to let all streams continue to completion. + internal static readonly PSFQuerySettings Default = new PSFQuerySettings { OnStreamEnded = (unusedPsf, unusedIndex) => true }; } } diff --git a/cs/src/core/Index/PSF/RecordIterator.cs b/cs/src/core/Index/PSF/RecordIterator.cs index 08b3c489e..7c7dd0e0a 100644 --- a/cs/src/core/Index/PSF/RecordIterator.cs +++ b/cs/src/core/Index/PSF/RecordIterator.cs @@ -5,208 +5,377 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Threading.Tasks; namespace FASTER.core.Index.PSF { /// - /// A single PSF's stream of recordIds + /// Base class to implement common functionality between sync and async versions of RecordIterator. /// - internal class RecordIterator where TRecordId : IComparable + internal abstract class RecordIteratorBase where TRecordId : IComparable { - private readonly IEnumerator enumerator; - - // Unfortunately we must sort to do the merge. - internal RecordIterator(IEnumerable enumer) => this.enumerator = enumer.OrderBy(rec => rec).GetEnumerator(); + internal IPSF psf; + internal int psfIndex; + protected readonly TEnumerator enumerator; - internal bool Next() + protected RecordIteratorBase(IPSF psf, int psfIndex, TEnumerator enumer) { - if (!this.IsDone) - this.IsDone = !this.enumerator.MoveNext(); - return !this.IsDone; + this.psf = psf; + this.psfIndex = psfIndex; + this.enumerator = enumer; } internal bool IsDone { get; set; } - internal TRecordId Current => this.enumerator.Current; + internal abstract TRecordId Current { get; } internal void GetIfLower(ref TRecordId currentLowest) { - if (!this.IsDone && this.enumerator.Current.CompareTo(currentLowest) < 0) - currentLowest = this.enumerator.Current; + if (!this.IsDone && this.Current.CompareTo(currentLowest) < 0) + currentLowest = this.Current; } - internal bool IsMatch(TRecordId recordId) => !this.IsDone && this.enumerator.Current.CompareTo(recordId) == 0; + internal bool IsMatch(TRecordId recordId) => !this.IsDone && this.Current.CompareTo(recordId) == 0; + + public override string ToString() => $"psfIdx {this.psfIndex}, current {this.Current}, isDone {this.IsDone}"; } /// - /// A single TPSFKey type's vector of its PSFs' streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// A single PSF's stream of recordIds /// - internal class KeyTypeRecordIterator where TRecordId : IComparable + internal class RecordIterator : RecordIteratorBase> where TRecordId : IComparable { - private struct PsfRecords + internal RecordIterator(IPSF psf, int psfIndex, IEnumerator enumerator) : base(psf, psfIndex, enumerator) { } + + internal bool Next() { - internal IPSF psf; - internal RecordIterator iterator; + if (!this.IsDone) + this.IsDone = !this.enumerator.MoveNext(); + return !this.IsDone; } - private readonly int keyTypeOrdinal; - private readonly PsfRecords[] psfRecords; - private readonly PSFQuerySettings querySettings; - private int numDone; + internal override TRecordId Current => this.enumerator.Current; + } - internal KeyTypeRecordIterator(int keyTypeOrd, IPSF psf1, IEnumerable psfRecordEnumerator1, PSFQuerySettings querySettings) - : this(keyTypeOrd, new[] { new PsfRecords { psf = psf1, iterator = new RecordIterator(psfRecordEnumerator1) } }, querySettings) - { } +#if DOTNETCORE + /// + /// A single PSF's async stream of recordIds + /// + internal class AsyncRecordIterator : RecordIteratorBase> where TRecordId : IComparable + { + internal AsyncRecordIterator(IPSF psf, int psfIndex, IAsyncEnumerator enumerator) : base(psf, psfIndex, enumerator) { } - internal KeyTypeRecordIterator(int keyTypeOrd, IEnumerable<(IPSF psf, IEnumerable psfRecEnum)> queryResults, PSFQuerySettings querySettings) - : this(keyTypeOrd, queryResults.Select(tup => new PsfRecords { psf = tup.psf, iterator = new RecordIterator(tup.psfRecEnum) }).ToArray(), querySettings) - { } + internal async Task NextAsync() + { + if (!this.IsDone) + this.IsDone = !await this.enumerator.MoveNextAsync(); + return !this.IsDone; + } + + internal override TRecordId Current => this.enumerator.Current; + } +#endif // DOTNETCORE + + /// + /// Base class to implement common functionality between sync and async versions of KeyTypeRecordIterator. + /// + internal class KeyTypeRecordIteratorBase where TRecordId : IComparable + { + private readonly int keyTypeOrdinal; + protected readonly RecordIteratorBase[] psfRecordIterators; + protected readonly PSFQuerySettings querySettings; + private int numDone; - private KeyTypeRecordIterator(int keyTypeOrd, PsfRecords[] psfRecs, PSFQuerySettings querySettings) + protected KeyTypeRecordIteratorBase(int keyTypeOrd, RecordIteratorBase[] psfRecEnums, PSFQuerySettings querySettings) { this.keyTypeOrdinal = keyTypeOrd; - this.psfRecords = psfRecs; + this.psfRecordIterators = psfRecEnums; this.querySettings = querySettings; } - internal int Count => this.psfRecords.Length; + internal int Count => this.psfRecordIterators.Length; internal bool IsDone => this.numDone == this.Count; [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void GetIfLower(RecordIterator recordIter, ref TRecordId currentLowest, ref bool isFirst) + protected static (bool @continue, TRecordId lowest, bool first) GetIfLower(RecordIteratorBase recordIter, TRecordId currentLowest, bool isFirst) { if (recordIter.IsDone) - return; + return (true, currentLowest, isFirst); if (isFirst) - { - currentLowest = recordIter.Current; - isFirst = false; - return; - } + return (true, recordIter.Current, false); recordIter.GetIfLower(ref currentLowest); + return (true, currentLowest, false); } - internal bool Initialize(ref TRecordId currentLowest, ref bool isFirst) + protected bool ContinueOnEOS(RecordIteratorBase recordIter) { - foreach (var (recordIter, psfIndex) in this.psfRecords.Select((item, index) => (item.iterator, index))) - { - if (!this.Next(recordIter, psfIndex) || querySettings.IsCanceled) - return false; - this.GetIfLower(recordIter, ref currentLowest, ref isFirst); - } - return true; + ++this.numDone; + return !this.querySettings.CancelOnEOS(recordIter.psf, (this.keyTypeOrdinal, recordIter.psfIndex)); } - private bool Next(RecordIterator recordIter, int psfIndex) + internal void MarkMatchIndicators(TRecordId currentLowest, bool[] matchIndicators) { - if (!recordIter.Next()) - { - ++this.numDone; - if (this.querySettings.CancelOnEOS(this.psfRecords[psfIndex].psf, (this.keyTypeOrdinal, psfIndex))) - return false; - } - return true; + foreach (var (recordIter, psfIndex) in this.psfRecordIterators.Select((item, index) => (item, index))) + matchIndicators[psfIndex] = recordIter.IsMatch(currentLowest); } - internal bool GetNextLowest(TRecordId previousLowest, ref TRecordId currentLowest, ref bool isFirst) + internal IEnumerable> GetEnumeratorsMatchingPrevLowest(TRecordId previousLowest) { - foreach (var (recordIter, psfIndex) in this.psfRecords.Select((item, index) => (item.iterator, index))) + foreach (var (recordIter, psfIndex) in this.psfRecordIterators.Select((item, index) => (item, index))) { if (recordIter.IsDone) continue; - if (querySettings.IsCanceled) - return false; - if (recordIter.IsMatch(previousLowest) && !this.Next(recordIter, psfIndex)) - return false; - this.GetIfLower(recordIter, ref currentLowest, ref isFirst); + if (this.querySettings.IsCanceled) + yield break; + if (recordIter.IsMatch(previousLowest)) + yield return recordIter; } - return true; } - internal void MarkMatchIndicators(TRecordId currentLowest, bool[] matchIndicators) + public override string ToString() => $"keyTypeOrd {this.keyTypeOrdinal}, count {this.Count}, isDone {this.IsDone}"; + } + + /// + /// A single TPSFKey type's vector of its PSFs' streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// + internal class KeyTypeRecordIterator : KeyTypeRecordIteratorBase> where TRecordId : IComparable + { + internal KeyTypeRecordIterator(int keyTypeOrd, IPSF psf1, IEnumerator psfRecordEnumerator1, PSFQuerySettings querySettings) + : base(keyTypeOrd, new[] { new RecordIterator(psf1, 0, psfRecordEnumerator1) }, querySettings) + { } + + internal KeyTypeRecordIterator(int keyTypeOrd, IEnumerable<(IPSF psf, IEnumerator psfRecEnum)> queryResults, PSFQuerySettings querySettings) + : base(keyTypeOrd, queryResults.Select((tup, psfIdx) => new RecordIterator(tup.psf, psfIdx, tup.psfRecEnum)).ToArray(), querySettings) + { } + + #region Sync methods + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal (bool @continue, TRecordId lowest, bool first) Initialize(TRecordId currentLowest, bool isFirst) + => IterateAndGetIfLower(true, currentLowest, currentLowest, isFirst); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal (bool @continue, TRecordId lowest, bool first) GetNextLowest(TRecordId previousLowest, TRecordId currentLowest, bool isFirst) + => IterateAndGetIfLower(false, previousLowest, currentLowest, isFirst); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal (bool @continue, TRecordId lowest, bool first) IterateAndGetIfLower(bool isInit, TRecordId previousLowest, TRecordId currentLowest, bool isFirst) { - foreach (var (recordIter, psfIndex) in this.psfRecords.Select((item, index) => (item.iterator, index))) - matchIndicators[psfIndex] = recordIter.IsMatch(currentLowest); + var tuple = (true, currentLowest, isFirst); + foreach (var recordIter in this.psfRecordIterators.Where(iter => !iter.IsDone).Cast>()) + { + // If in initialization, always do the initial Next(); otherwise, advance the iterator if it matches the previous lowest record ID. + if (((isInit || recordIter.IsMatch(previousLowest)) && !recordIter.Next() && !ContinueOnEOS(recordIter)) || querySettings.IsCanceled) + return (false, currentLowest, isFirst); + tuple = GetIfLower(recordIter, tuple.currentLowest, tuple.isFirst); + } + return tuple; } + #endregion Sync methods } +#if DOTNETCORE // TODO: Refactor some of this cut and paste to a KeyTypeRecordIteratorBase templated on the record enumerable type /// - /// The complete query's PSFs' streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// A single TPSFKey type's vector of its PSFs' async streams of recordIds (each TPSFKey type may have multiple PSFs being queried). /// - internal class QueryRecordIterator where TRecordId : IComparable + internal class AsyncKeyTypeRecordIterator : KeyTypeRecordIteratorBase> where TRecordId : IComparable { - private readonly KeyTypeRecordIterator[] keyTypeRecordIterators; + internal AsyncKeyTypeRecordIterator(int keyTypeOrd, IPSF psf1, IAsyncEnumerator psfRecordEnumerator1, PSFQuerySettings querySettings) + : base(keyTypeOrd, new[] { new AsyncRecordIterator(psf1, 0, psfRecordEnumerator1) }, querySettings) + { } + + internal AsyncKeyTypeRecordIterator(int keyTypeOrd, IEnumerable<(IPSF psf, IAsyncEnumerator psfRecEnum)> queryResults, PSFQuerySettings querySettings) + : base(keyTypeOrd, queryResults.Select((tup, psfIdx) => new AsyncRecordIterator(tup.psf, psfIdx, tup.psfRecEnum)).ToArray(), querySettings) + { } + + #region Async methods + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal async Task<(bool @continue, TRecordId lowest, bool first)> InitializeAsync(TRecordId currentLowest, bool isFirst) + => await IterateAndGetIfLowerAsync(true, currentLowest, currentLowest, isFirst); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal async Task<(bool @continue, TRecordId lowest, bool first)> GetNextLowestAsync(TRecordId previousLowest, TRecordId currentLowest, bool isFirst) + => await IterateAndGetIfLowerAsync(false, previousLowest, currentLowest, isFirst); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal async Task<(bool @continue, TRecordId lowest, bool first)> IterateAndGetIfLowerAsync(bool isInit, TRecordId previousLowest, TRecordId currentLowest, bool isFirst) + { + var tuple = (true, currentLowest, isFirst); + foreach (var recordIter in this.psfRecordIterators.Where(iter => !iter.IsDone).Cast>()) + { + // If in initialization, always do the initial Next(); otherwise, advance the iterator if it matches the previous lowest record ID. + if (((isInit || recordIter.IsMatch(previousLowest)) && !await recordIter.NextAsync() && !ContinueOnEOS(recordIter)) || querySettings.IsCanceled) + return (false, currentLowest, isFirst); + tuple = GetIfLower(recordIter, tuple.currentLowest, tuple.isFirst); + } + return tuple; + } + #endregion Async methods + } +#endif // DOTNETCORE + + /// + /// Base class to implement common functionality between sync and async versions of KeyTypeRecordIterator. + /// + internal class QueryRecordIteratorBase where TRecordId : IComparable + { + protected readonly KeyTypeRecordIteratorBase[] keyTypeRecordIterators; private readonly bool[][] matchIndicators; private readonly PSFQuerySettings querySettings; private readonly Func callerLambda; + protected QueryRecordIteratorBase(KeyTypeRecordIteratorBase[] ktris, Func callerLambda, PSFQuerySettings querySettings) + { + this.keyTypeRecordIterators = ktris; + this.matchIndicators = this.keyTypeRecordIterators.Select(ktri => new bool[ktri.Count]).ToArray(); + this.callerLambda = callerLambda; + this.querySettings = querySettings; + } + + protected bool CallLambda(ref TRecordId current, out bool emit) + { + var allDone = true; + foreach (var (keyIter, keyIndex) in this.keyTypeRecordIterators.Select((iter, index) => (iter, index))) + { + keyIter.MarkMatchIndicators(current, this.matchIndicators[keyIndex]); + allDone &= keyIter.IsDone; + } + + allDone |= this.querySettings.IsCanceled; + emit = !allDone && this.callerLambda(this.matchIndicators); + return !allDone; + } + } + + /// + /// The complete query's PSFs' streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// + internal class QueryRecordIterator : QueryRecordIteratorBase> where TRecordId : IComparable + { + // Unfortunately we must sort to do the merge. + private static IEnumerator GetOrderedEnumerator(IEnumerable enumerable) => enumerable.OrderBy(rec => rec).GetEnumerator(); + internal QueryRecordIterator(IPSF psf1, IEnumerable keyRecords1, IPSF psf2, IEnumerable keyRecords2, Func callerLambda, PSFQuerySettings querySettings) - : this(new[] { - new KeyTypeRecordIterator(0, psf1, keyRecords1, querySettings), - new KeyTypeRecordIterator(1, psf2, keyRecords2, querySettings) + : base(new[] { + new KeyTypeRecordIterator(0, psf1, GetOrderedEnumerator(keyRecords1), querySettings), + new KeyTypeRecordIterator(1, psf2, GetOrderedEnumerator(keyRecords2), querySettings) }, callerLambda, querySettings) { } internal QueryRecordIterator(IPSF psf1, IEnumerable keyRecords1, IPSF psf2, IEnumerable keyRecords2, IPSF psf3, IEnumerable keyRecords3, Func callerLambda, PSFQuerySettings querySettings) - : this(new[] { - new KeyTypeRecordIterator(0, psf1, keyRecords1, querySettings), - new KeyTypeRecordIterator(1, psf2, keyRecords2, querySettings), - new KeyTypeRecordIterator(2, psf3, keyRecords3, querySettings) + : base(new[] { + new KeyTypeRecordIterator(0, psf1, GetOrderedEnumerator(keyRecords1), querySettings), + new KeyTypeRecordIterator(1, psf2, GetOrderedEnumerator(keyRecords2), querySettings), + new KeyTypeRecordIterator(2, psf3, GetOrderedEnumerator(keyRecords3), querySettings) }, callerLambda, querySettings) { } internal QueryRecordIterator(IEnumerable keyRecEnums)>> keyTypeQueryResultsEnum, Func callerLambda, PSFQuerySettings querySettings) - : this(keyTypeQueryResultsEnum.Select((ktqr, index) => new KeyTypeRecordIterator(index, ktqr, querySettings)).ToArray(), + : base(keyTypeQueryResultsEnum.Select((ktqr, index) => new KeyTypeRecordIterator(index, ktqr.Select(tuple => (tuple.psf, GetOrderedEnumerator(tuple.keyRecEnums))), querySettings)).ToArray(), callerLambda, querySettings) { } - private QueryRecordIterator(KeyTypeRecordIterator[] ktris, Func callerLambda, PSFQuerySettings querySettings) - { - this.keyTypeRecordIterators = ktris; - this.matchIndicators = this.keyTypeRecordIterators.Select(ktri => new bool[ktri.Count]).ToArray(); - this.callerLambda = callerLambda; - this.querySettings = querySettings; - } - + #region Sync methods internal IEnumerable Run() { - TRecordId current = default; - bool isFirst = true; - foreach (var keyIter in this.keyTypeRecordIterators) + // The tuple is necessary due to async prohibition of byref parameters. + (bool @continue, TRecordId current, bool isFirst) tuple = (true, default, true); + foreach (var keyIter in this.keyTypeRecordIterators.Cast>()) { - if (!keyIter.Initialize(ref current, ref isFirst)) + tuple = keyIter.Initialize(tuple.current, tuple.isFirst); + if (!tuple.@continue) yield break; } - while(true) + while (true) { - var allDone = true; - foreach (var (keyIter, keyIndex) in this.keyTypeRecordIterators.Select((iter, index) => (iter, index))) + if (!CallLambda(ref tuple.current, out bool emit)) + yield break; + if (emit) + yield return tuple.current; + + var prevLowest = tuple.current; + tuple.isFirst = true; + foreach (var keyIter in this.keyTypeRecordIterators.Cast>()) { - keyIter.MarkMatchIndicators(current, this.matchIndicators[keyIndex]); - allDone &= keyIter.IsDone; + // TODOperf: consider a PQ here. Given that we have to go through all matchIndicators anyway, at what number of streams would the additional complexity improve speed? + tuple = keyIter.GetNextLowest(prevLowest, tuple.current, tuple.isFirst); + if (!tuple.@continue) + yield break; } + } + } + #endregion Sync methods + } + +#if DOTNETCORE + /// + /// The complete query's PSFs' async streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// + internal class AsyncQueryRecordIterator : QueryRecordIteratorBase> where TRecordId : IComparable + { + // Unfortunately we must sort to do the merge. + private static IAsyncEnumerator GetOrderedEnumerator(IAsyncEnumerable enumerable) => enumerable.OrderBy(rec => rec).GetAsyncEnumerator(); - if (allDone || this.querySettings.IsCanceled) + internal AsyncQueryRecordIterator(IPSF psf1, IAsyncEnumerable keyRecords1, IPSF psf2, IAsyncEnumerable keyRecords2, + Func callerLambda, PSFQuerySettings querySettings) + : base(new[] { + new AsyncKeyTypeRecordIterator(0, psf1, GetOrderedEnumerator(keyRecords1), querySettings), + new AsyncKeyTypeRecordIterator(1, psf2, GetOrderedEnumerator(keyRecords2), querySettings) + }, callerLambda, querySettings) + { } + + internal AsyncQueryRecordIterator(IPSF psf1, IAsyncEnumerable keyRecords1, IPSF psf2, IAsyncEnumerable keyRecords2, + IPSF psf3, IAsyncEnumerable keyRecords3, + Func callerLambda, PSFQuerySettings querySettings) + : base(new[] { + new AsyncKeyTypeRecordIterator(0, psf1, GetOrderedEnumerator(keyRecords1), querySettings), + new AsyncKeyTypeRecordIterator(1, psf2, GetOrderedEnumerator(keyRecords2), querySettings), + new AsyncKeyTypeRecordIterator(2, psf3, GetOrderedEnumerator(keyRecords3), querySettings) + }, callerLambda, querySettings) + { } + + internal AsyncQueryRecordIterator(IEnumerable keyRecEnums)>> keyTypeQueryResultsEnum, + Func callerLambda, PSFQuerySettings querySettings) + : base(keyTypeQueryResultsEnum.Select((ktqr, index) => new AsyncKeyTypeRecordIterator(index, ktqr.Select(tuple => (tuple.psf, GetOrderedEnumerator(tuple.keyRecEnums))), querySettings)).ToArray(), + callerLambda, querySettings) + { } + + #region Sync methods + internal async IAsyncEnumerable Run() + { + // The tuple is necessary due to async prohibition of byref parameters. + (bool @continue, TRecordId current, bool isFirst) tuple = (true, default, true); + foreach (var keyIter in this.keyTypeRecordIterators.Cast>()) + { + tuple = await keyIter.InitializeAsync(tuple.current, tuple.isFirst); + if (!tuple.@continue) yield break; + } - if (this.callerLambda(this.matchIndicators)) - yield return current; + while (true) + { + if (!CallLambda(ref tuple.current, out bool emit)) + yield break; + if (emit) + yield return tuple.current; - var prevLowest = current; - isFirst = true; - foreach (var keyIter in this.keyTypeRecordIterators) + var prevLowest = tuple.current; + tuple.isFirst = true; + foreach (var keyIter in this.keyTypeRecordIterators.Cast>()) { // TODOperf: consider a PQ here. Given that we have to go through all matchIndicators anyway, at what number of streams would the additional complexity improve speed? - if (!keyIter.GetNextLowest(prevLowest, ref current, ref isFirst)) + tuple = await keyIter.GetNextLowestAsync(prevLowest, tuple.current, tuple.isFirst); + if (!tuple.@continue) yield break; } } } + #endregion Sync methods } +#endif // DOTNETCORE } From 9163f1f25fc3f2a227c5863761d7313d291216e5 Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Sun, 23 Aug 2020 18:36:45 -0700 Subject: [PATCH 14/19] Prep for functions-in-session: - Convert secondary FKV Key type from CompositeKey to TPSFKey - Remove IKeyAccessor as we can now use KeyAccessor directly - Also cleans up managing the single query key vs composite key handling - Add PsfQueryKeyContainer - Limit # of PSFs per group to 255 --- .../FasterPSFSample/FasterPSFSample.cs | 7 ++ cs/src/core/Allocator/AllocatorBase.cs | 4 +- cs/src/core/Index/FASTER/FASTER.cs | 11 +-- cs/src/core/Index/FASTER/FASTERBase.cs | 2 +- cs/src/core/Index/FASTER/FASTERImpl.cs | 1 - cs/src/core/Index/PSF/CompositeKey.cs | 36 +++++-- .../Index/PSF/FasterPSFContextOperations.cs | 4 +- cs/src/core/Index/PSF/FasterPSFImpl.cs | 60 ++++++++---- .../Index/PSF/FasterPSFSessionOperations.cs | 8 +- cs/src/core/Index/PSF/GroupKeys.cs | 17 ++-- cs/src/core/Index/PSF/KeyAccessor.cs | 94 ++++++------------- cs/src/core/Index/PSF/KeyPointer.cs | 16 +++- cs/src/core/Index/PSF/PSFContext.cs | 12 --- cs/src/core/Index/PSF/PSFFunctions.cs | 26 ++--- cs/src/core/Index/PSF/PSFGroup.cs | 76 +++++++-------- cs/src/core/Index/PSF/PSFInput.cs | 18 ++-- cs/src/core/Index/PSF/PSFManager.cs | 8 ++ cs/src/core/Index/PSF/PSFOutput.cs | 11 ++- cs/src/core/Index/PSF/PSFResultFlags.cs | 2 +- cs/src/core/Index/PSF/RecordIterator.cs | 2 +- 20 files changed, 210 insertions(+), 205 deletions(-) delete mode 100644 cs/src/core/Index/PSF/PSFContext.cs diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs index 5e48099fb..5a8500eea 100644 --- a/cs/playground/FasterPSFSample/FasterPSFSample.cs +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -66,9 +66,16 @@ internal async static Task RunSample>> "); Console.WriteLine(ok ? "Passed! All operations succeeded" : "*** Failed! *** One or more operations did not succeed"); Console.WriteLine(); + if (Debugger.IsAttached) + { + Console.Write("Press ENTER to close this window . . ."); + Console.ReadKey(); + } } finally { diff --git a/cs/src/core/Allocator/AllocatorBase.cs b/cs/src/core/Allocator/AllocatorBase.cs index 1ab46d1e1..fb2b99ca1 100644 --- a/cs/src/core/Allocator/AllocatorBase.cs +++ b/cs/src/core/Allocator/AllocatorBase.cs @@ -53,7 +53,7 @@ public unsafe abstract partial class AllocatorBase : IDisposable /// protected readonly IFasterEqualityComparer comparer; - internal IKeyAccessor PsfKeyAccessor; + internal KeyAccessor PsfKeyAccessor; #region Protected size definitions /// @@ -1478,7 +1478,7 @@ private void AsyncGetFromDiskCallback(uint errorCode, uint numBytes, NativeOverl var isEqualKey = ctx.request_key is null || (this.PsfKeyAccessor is null ? comparer.Equals(ref ctx.request_key.Get(), ref GetContextRecordKey(ref ctx)) - : this.PsfKeyAccessor.EqualsAtRecordAddress(ref ctx.request_key.Get(), (long)record)); + : this.PsfKeyAccessor.EqualsAtRecordAddress(ref KeyPointer.CastFromKeyRef(ref ctx.request_key.Get()), (long)record)); if (isEqualKey) { // ctx.record = result.record; diff --git a/cs/src/core/Index/FASTER/FASTER.cs b/cs/src/core/Index/FASTER/FASTER.cs index eb2e34ddf..cf2561a1c 100644 --- a/cs/src/core/Index/FASTER/FASTER.cs +++ b/cs/src/core/Index/FASTER/FASTER.cs @@ -199,10 +199,9 @@ public bool TakeFullCheckpoint(out Guid token, long targetVersion = -1) var result = StartStateMachine(new FullCheckpointStateMachine(backend, targetVersion)); token = _hybridLogCheckpointToken; - // Do not return the PSF token here. TODO Separate Tasks? + // Do not return the PSF token here. TODO: Handle failure of PSFManager.TakeFullCheckpoint if (result && this.PSFManager.HasPSFs) result &= this.PSFManager.TakeFullCheckpoint(); - return result; } @@ -242,7 +241,7 @@ public bool TakeIndexCheckpoint(out Guid token) var result = StartStateMachine(new IndexSnapshotStateMachine()); token = _indexCheckpointToken; - // Do not return the PSF token here. TODO Separate Tasks? + // Do not return the PSF token here. TODO: Handle failure of PSFManager.TakeIndexCheckpoint if (result && this.PSFManager.HasPSFs) result &= this.PSFManager.TakeIndexCheckpoint(); return result; @@ -261,7 +260,7 @@ public bool TakeHybridLogCheckpoint(out Guid token, long targetVersion = -1) var result = StartStateMachine(new HybridLogCheckpointStateMachine(backend, targetVersion)); token = _hybridLogCheckpointToken; - // Do not return the PSF token here. TODO Separate Tasks? + // Do not return the PSF token here. TODO: Handle failure of PSFManager.TakeHybridLogCheckpoint if (result && this.PSFManager.HasPSFs) result &= this.PSFManager.TakeHybridLogCheckpoint(); return result; @@ -296,7 +295,7 @@ public void Recover() { InternalRecoverFromLatestCheckpoints(); - if (this.PSFManager.HasPSFs) // TODO Separate Tasks? + if (this.PSFManager.HasPSFs) // TODO: Handle failure of PSFManager.Recovery this.PSFManager.Recover(); } @@ -342,7 +341,7 @@ public async ValueTask CompleteCheckpointAsync(CancellationToken token = default await ThreadStateMachineStep(null, NullFasterSession.Instance, true, token); } - if (this.PSFManager.HasPSFs) // TODO Separate Task? + if (this.PSFManager.HasPSFs) // TODO: Do in parallel and handle failure of PSFManager.CompleteCheckpointAsync await this.PSFManager.CompleteCheckpointAsync(); } diff --git a/cs/src/core/Index/FASTER/FASTERBase.cs b/cs/src/core/Index/FASTER/FASTERBase.cs index 83fff92b8..e6a1e4946 100644 --- a/cs/src/core/Index/FASTER/FASTERBase.cs +++ b/cs/src/core/Index/FASTER/FASTERBase.cs @@ -76,7 +76,7 @@ internal static class Constants public const int kFirstValidAddress = 64; public const long kInvalidPsfGroupId = -1; - public const int kInvalidPsfOrdinal = -1; + public const int kInvalidPsfOrdinal = 255; // 0-based ordinals; this is also the max count } [StructLayout(LayoutKind.Explicit, Size = Constants.kEntriesPerBucket * 8)] diff --git a/cs/src/core/Index/FASTER/FASTERImpl.cs b/cs/src/core/Index/FASTER/FASTERImpl.cs index e33b98872..debaa7c2a 100644 --- a/cs/src/core/Index/FASTER/FASTERImpl.cs +++ b/cs/src/core/Index/FASTER/FASTERImpl.cs @@ -2149,7 +2149,6 @@ private bool ReadFromCache(ref Key key, ref long logicalAddress, ref long physic entry.word = logicalAddress; if (!entry.ReadCache) break; physicalAddress = readcache.GetPhysicalAddress(logicalAddress & ~Constants.kReadCacheBitMask); - // TODO: Update latestRecordVersion? } physicalAddress = 0; return false; diff --git a/cs/src/core/Index/PSF/CompositeKey.cs b/cs/src/core/Index/PSF/CompositeKey.cs index 43fe18950..d7f8920ca 100644 --- a/cs/src/core/Index/PSF/CompositeKey.cs +++ b/cs/src/core/Index/PSF/CompositeKey.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -using System; using System.Runtime.CompilerServices; namespace FASTER.core @@ -29,13 +28,32 @@ internal ref KeyPointer GetKeyPointerRef(int psfOrdinal, int keyPointer /// Get a reference to the key for the PSF identified by psfOrdinal. /// /// The ordinal of the PSF in its parent PSFGroup - /// Size of the KeyPointer{TPSFKey} struct + /// Size of the struct /// A reference to the key for the PSF identified by psfOrdinal. [MethodImpl(MethodImplOptions.AggressiveInlining)] internal ref TPSFKey GetKeyRef(int psfOrdinal, int keyPointerSize) => ref GetKeyPointerRef(psfOrdinal, keyPointerSize).Key; - internal class VarLenLength : IVariableLengthStruct> + /// + /// Returns a reference to the CompositeKey from a reference to the first + /// + /// A reference to the first , typed as TPSFKey + /// Used when converting the CompositeKey to/from the TPSFKey type for secondary FKV operations + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static ref CompositeKey CastFromFirstKeyPointerRefAsKeyRef(ref TPSFKey firstKeyPointerRef) + => ref Unsafe.AsRef>((byte*)Unsafe.AsPointer(ref firstKeyPointerRef)); + + /// + /// Converts this CompositeKey reference to a reference to the first , typed as TPSFKey. + /// + /// Used when converting the CompositeKey to/from the TPSFKey type for secondary FKV operations + /// A reference to the first , typed as TPSFKey + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ref TPSFKey CastToFirstKeyPointerRefAsKeyRef() + => ref Unsafe.AsRef((byte*)Unsafe.AsPointer(ref this)); + + internal class VarLenLength : IVariableLengthStruct { private readonly int size; @@ -43,19 +61,19 @@ internal class VarLenLength : IVariableLengthStruct> public int GetInitialLength() => this.size; - public int GetLength(ref CompositeKey _) => this.size; + public int GetLength(ref TPSFKey _) => this.size; } /// /// This is the unused key comparer passed to the secondary FasterKV /// - internal class UnusedKeyComparer : IFasterEqualityComparer> + internal class UnusedKeyComparer : IFasterEqualityComparer { - public long GetHashCode64(ref CompositeKey cKey) - => throw new PSFInternalErrorException("Must use KeyAccessor instead"); + public long GetHashCode64(ref TPSFKey cKey) + => throw new PSFInternalErrorException("Must use KeyAccessor instead (psfOrdinal is required)"); - public bool Equals(ref CompositeKey cKey1, ref CompositeKey cKey2) - => throw new PSFInternalErrorException("Must use KeyAccessor instead"); + public bool Equals(ref TPSFKey cKey1, ref TPSFKey cKey2) + => throw new PSFInternalErrorException("Must use KeyAccessor instead (psfOrdinal is required)"); } } } diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs index bb061d388..3ef3fb173 100644 --- a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -137,7 +137,7 @@ internal Status ContextPsfUpdate(), ref value, ref input, + var internalStatus = this.PsfInternalInsert(ref groupKeys.CastToKeyRef(), ref value, ref input, ref pcontext, fasterSession, sessionCtx, serialNo); Status status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND ? (Status)internalStatus @@ -161,7 +161,7 @@ private Status PsfRcuInsert(GroupKeys gro var psfInput = (IPSFInput)input; unsafe { psfInput.SetFlags(groupKeys.ResultFlags); } psfInput.IsDelete = false; - var internalStatus = this.PsfInternalInsert(ref groupKeys.GetCompositeKeyRef(), ref value, ref input, + var internalStatus = this.PsfInternalInsert(ref groupKeys.CastToKeyRef(), ref value, ref input, ref pcontext, fasterSession, sessionCtx, serialNo); return internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND ? (Status)internalStatus diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index e7742257a..4cda9e5c7 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -18,11 +18,11 @@ public unsafe partial class FasterKV : FasterBase, IFasterKV PsfKeyAccessor => this.hlog.PsfKeyAccessor; + internal KeyAccessor PsfKeyAccessor => this.hlog.PsfKeyAccessor; internal bool ImplmentsPSFs => !(this.PsfKeyAccessor is null); - bool ScanQueryChain(ref long logicalAddress, ref Key queryKey, ref int latestRecordVersion) + bool ScanQueryChain(ref long logicalAddress, ref KeyPointer queryKeyPointer, ref int latestRecordVersion) { long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); var recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); @@ -31,7 +31,7 @@ bool ScanQueryChain(ref long logicalAddress, ref Key queryKey, ref int latestRec while (true) { - if (this.PsfKeyAccessor.EqualsAtKeyAddress(ref queryKey, physicalAddress)) + if (this.PsfKeyAccessor.EqualsAtKeyAddress(ref queryKeyPointer, physicalAddress)) { PsfTrace($" / {logicalAddress}"); return true; @@ -57,9 +57,27 @@ private void PsfTraceLine(string message = null) if (this.ImplmentsPSFs) Console.WriteLine(message ?? string.Empty); } + // PsfKeyContainer is necessary because VarLenBlittableAllocator.GetKeyContainer will use the size of the full + // composite key (KeyPointerSize * PsfCount), but the query key has only one KeyPointer. + private class PsfQueryKeyContainer : IHeapContainer + { + private readonly SectorAlignedMemory mem; + + public unsafe PsfQueryKeyContainer(ref Key key, KeyAccessor keyAccessor, SectorAlignedBufferPool pool) + { + var len = keyAccessor.KeyPointerSize; + this.mem = pool.Get(len); + Buffer.MemoryCopy(Unsafe.AsPointer(ref key), mem.GetValidPointer(), len, len); + } + + public unsafe ref Key Get() => ref Unsafe.AsRef(this.mem.GetValidPointer()); + + public void Dispose() => this.mem.Return(); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal OperationStatus PsfInternalReadKey( - ref Key queryKey, ref PSFReadArgs psfArgs, + ref Key queryKeyPointerRefAsKeyRef, ref PSFReadArgs psfArgs, ref PendingContext pendingContext, FasterSession fasterSession, FasterExecutionContext sessionCtx, long lsn) @@ -73,8 +91,9 @@ internal OperationStatus PsfInternalReadKey queryKeyPointer = ref KeyPointer.CastFromKeyRef(ref queryKeyPointerRefAsKeyRef); - var hash = this.PsfKeyAccessor.GetHashCode64(ref queryKey, psfInput.PsfOrdinal, isQuery:true); // the queryKey has only one key + var hash = this.PsfKeyAccessor.GetHashCode64(ref queryKeyPointer); var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); if (sessionCtx.phase != Phase.REST) @@ -86,7 +105,7 @@ internal OperationStatus PsfInternalReadKey= hlog.HeadAddress) { - if (!ScanQueryChain(ref logicalAddress, ref psfInput.QueryKeyRef, ref latestRecordVersion)) + if (!ScanQueryChain(ref logicalAddress, ref queryKeyPointer, ref latestRecordVersion)) goto ProcessAddress; // RECORD_ON_DISK or not found } } @@ -176,7 +195,7 @@ ref hlog.GetValue(recordAddress), CreatePendingContext: { pendingContext.type = OperationType.PSF_READ_KEY; - pendingContext.key = hlog.GetKeyContainer(ref queryKey); + pendingContext.key = new PsfQueryKeyContainer(ref queryKeyPointerRefAsKeyRef, this.PsfKeyAccessor, this.hlog.bufferPool); pendingContext.input = default; pendingContext.output = default; pendingContext.userContext = default; @@ -234,7 +253,7 @@ ref readcache.GetValue(physicalAddress), if (logicalAddress >= hlog.HeadAddress) { - if (this.ImplmentsPSFs && !ScanQueryChain(ref logicalAddress, ref psfInput.QueryKeyRef, ref latestRecordVersion)) + if (this.ImplmentsPSFs && !ScanQueryChain(ref logicalAddress, ref KeyPointer.CastFromKeyRef(ref psfInput.QueryKeyRef), ref latestRecordVersion)) { goto ProcessAddress; // RECORD_ON_DISK or not found } @@ -284,7 +303,9 @@ ref hlog.GetValue(recordAddress), CreatePendingContext: { pendingContext.type = OperationType.PSF_READ_ADDRESS; - pendingContext.key = this.ImplmentsPSFs ? hlog.GetKeyContainer(ref psfInput.QueryKeyRef) : default; + pendingContext.key = this.ImplmentsPSFs + ? new PsfQueryKeyContainer(ref psfInput.QueryKeyRef, this.PsfKeyAccessor, this.hlog.bufferPool) + : default; pendingContext.input = default; pendingContext.output = default; pendingContext.userContext = default; @@ -312,7 +333,7 @@ unsafe struct CASHelper [MethodImpl(MethodImplOptions.AggressiveInlining)] internal OperationStatus PsfInternalInsert( - ref Key compositeKey, ref Value value, ref Input input, + ref Key firstKeyPointerRefAsKeyRef, ref Value value, ref Input input, ref PendingContext pendingContext, FasterSession fasterSession, FasterExecutionContext sessionCtx, long lsn) @@ -322,12 +343,12 @@ internal OperationStatus PsfInternalInsert; + ref CompositeKey compositeKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref firstKeyPointerRefAsKeyRef); // Update the KeyPointer links for chains with IsNullAt false (indicating a match with the // corresponding PSF) to point to the previous records for all keys in the composite key. // Note: We're not checking for a previous occurrence of the input value (the recordId) because // we are doing insert only here; the update part of upsert is done in PsfInternalUpdate. - // TODO: Limit size of stackalloc based on # of PSFs. var psfCount = this.PsfKeyAccessor.KeyCount; CASHelper* casHelpers = stackalloc CASHelper[psfCount]; PsfTrace($"Insert: {this.PsfKeyAccessor.GetString(ref compositeKey)} | rId {value} |"); @@ -344,7 +365,7 @@ internal OperationStatus PsfInternalInsert> Constants.kHashTagShift); if (sessionCtx.phase != Phase.REST) @@ -407,15 +428,16 @@ internal OperationStatus PsfInternalInsert storedKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref storedFirstKeyPointerRefAsKeyRef); + hlog.ShallowCopy(ref firstKeyPointerRefAsKeyRef, ref storedFirstKeyPointerRefAsKeyRef); + hlog.ShallowCopy(ref value, ref hlog.GetValue(newPhysicalAddress)); PsfTraceLine(); newLogicalAddress += RecordInfo.GetLength(); @@ -480,7 +502,7 @@ internal OperationStatus PsfInternalInsert> ReturnProviderDatas(IEnum // TODOerr: Handle error status from PsfReadAddress } while (providerDatas.TryDequeue(out var providerData)) + { + // TODO: Liveness check via Read yield return providerData; + } } this.CompletePending(spinWait: true); @@ -161,8 +164,11 @@ internal async IAsyncEnumerable> ReturnProvider await foreach (var logicalAddress in logicalAddresses) { var providerData = await this.CreateProviderDataAsync(logicalAddress, providerDatas, querySettings); - if (!(providerData is null)) + { + // TODO: No Async query ops if threadaffinitized + // TODO: Liveness check via ReadAsync yield return providerData; + } } } #endif diff --git a/cs/src/core/Index/PSF/GroupKeys.cs b/cs/src/core/Index/PSF/GroupKeys.cs index 408c76962..9654ef8bc 100644 --- a/cs/src/core/Index/PSF/GroupKeys.cs +++ b/cs/src/core/Index/PSF/GroupKeys.cs @@ -18,14 +18,15 @@ internal void Set(SectorAlignedMemory keyMem, SectorAlignedMemory flagsMem) this.flagsMem = flagsMem; } - internal unsafe ref TCompositeKey GetCompositeKeyRef() - => ref Unsafe.AsRef(this.compositeKeyMem.GetValidPointer()); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal unsafe ref TCompositeOrIndividualKey CastToKeyRef() + => ref Unsafe.AsRef(this.compositeKeyMem.GetValidPointer()); internal unsafe PSFResultFlags* ResultFlags => (PSFResultFlags*)this.flagsMem.GetValidPointer(); - internal unsafe bool IsNullAt(int ordinal) => (this.ResultFlags + ordinal)->HasFlag(PSFResultFlags.IsNull); - public unsafe bool IsUnlinkOldAt(int ordinal) => (this.ResultFlags + ordinal)->HasFlag(PSFResultFlags.UnlinkOld); - public unsafe bool IsLinkNewAt(int ordinal) => (this.ResultFlags + ordinal)->HasFlag(PSFResultFlags.LinkNew); + internal unsafe bool IsNullAt(int psfOrdinal) => (this.ResultFlags + psfOrdinal)->HasFlag(PSFResultFlags.IsNull); + public unsafe bool IsUnlinkOldAt(int psfOrdinal) => (this.ResultFlags + psfOrdinal)->HasFlag(PSFResultFlags.UnlinkOld); + public unsafe bool IsLinkNewAt(int psfOrdinal) => (this.ResultFlags + psfOrdinal)->HasFlag(PSFResultFlags.LinkNew); public unsafe bool HasChanges => this.ResultFlags->HasFlag(PSFResultFlags.UnlinkOld) || this.ResultFlags->HasFlag(PSFResultFlags.LinkNew); @@ -56,11 +57,9 @@ internal GroupKeysPair(long id) internal bool HasAddress => this.LogicalAddress != Constants.kInvalidAddress; - internal ref TCompositeKey GetBeforeKey() - => ref this.Before.GetCompositeKeyRef(); + internal ref TCompositeKey GetBeforeKey() => ref this.Before.CastToKeyRef(); - internal ref TCompositeKey GetAfterKey() - => ref this.After.GetCompositeKeyRef(); + internal ref TCompositeKey GetAfterKey() => ref this.After.CastToKeyRef(); internal bool HasChanges => this.After.HasChanges; diff --git a/cs/src/core/Index/PSF/KeyAccessor.cs b/cs/src/core/Index/PSF/KeyAccessor.cs index ca10a4876..0f4e2749c 100644 --- a/cs/src/core/Index/PSF/KeyAccessor.cs +++ b/cs/src/core/Index/PSF/KeyAccessor.cs @@ -6,41 +6,12 @@ namespace FASTER.core { - /// - /// Internal interface to bridge the generic definition for composite key type. - /// - /// - internal interface IKeyAccessor - { - int KeyCount { get; } - - int KeyPointerSize { get; } - - long GetPrevAddress(long physicalAddress); - - void SetPrevAddress(ref TCompositeKey key, int psfOrdinal, long prevAddress); - - long GetRecordAddressFromKeyPhysicalAddress(long physicalAddress); - - long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, int psfOrdinal); - - long GetKeyAddressFromRecordPhysicalAddress(long physicalAddress, int psfOrdinal); - - long GetHashCode64(ref TCompositeKey key, int psfOrdinal, bool isQuery); - - bool EqualsAtKeyAddress(ref TCompositeKey queryKey, long physicalAddress); - - bool EqualsAtRecordAddress(ref TCompositeKey queryKey, long physicalAddress); - - string GetString(ref TCompositeKey key, int psfOrdinal = -1); - } - /// /// Provides access to the internals that are hidden behind /// the Key typeparam of the secondary FasterKV. /// /// The type of the Key returned by a PSF function - internal unsafe class KeyAccessor : IKeyAccessor> + internal unsafe class KeyAccessor where TPSFKey : new() { private readonly IFasterEqualityComparer userComparer; @@ -57,7 +28,7 @@ internal KeyAccessor(IFasterEqualityComparer userComparer, int keyCount public int KeyPointerSize { get; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetPrevAddress(long physicalAddress) + public long GetPrevAddress(long physicalAddress) => GetKeyPointerRef(physicalAddress).PrevAddress; [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -66,47 +37,38 @@ public void SetPrevAddress(ref CompositeKey key, int psfOrdinal, long p [MethodImpl(MethodImplOptions.AggressiveInlining)] public long GetRecordAddressFromKeyPhysicalAddress(long physicalAddress) - => physicalAddress - this.GetKeyPointerRef(physicalAddress).PsfOrdinal * this.KeyPointerSize - RecordInfo.GetLength(); // TODO: Assumes all PSFs are present + => physicalAddress - this.GetKeyPointerRef(physicalAddress).PsfOrdinal * this.KeyPointerSize - RecordInfo.GetLength(); // Note: Assumes all PSFs are present [MethodImpl(MethodImplOptions.AggressiveInlining)] public long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, int psfOrdinal) - => logicalAddress - psfOrdinal * this.KeyPointerSize - RecordInfo.GetLength(); // TODO: Assumes all PSFs are present + => logicalAddress - psfOrdinal * this.KeyPointerSize - RecordInfo.GetLength(); // Note: Assumes all PSFs are present [MethodImpl(MethodImplOptions.AggressiveInlining)] public long GetKeyAddressFromRecordPhysicalAddress(long physicalAddress, int psfOrdinal) - => physicalAddress + RecordInfo.GetLength() + psfOrdinal * this.KeyPointerSize; // TODO: Assumes all PSFs are present + => physicalAddress + RecordInfo.GetLength() + psfOrdinal * this.KeyPointerSize; // Note: Assumes all PSFs are present [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetHashCode64(ref CompositeKey key, int psfOrdinal, bool isQuery) + public long GetHashCode64(ref KeyPointer keyPointer) + => Utility.GetHashCode(this.userComparer.GetHashCode64(ref keyPointer.Key)) ^ Utility.GetHashCode(keyPointer.PsfOrdinal + 1); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetHashCode64(ref CompositeKey key, int psfOrdinal) { - ref KeyPointer keyPointer = ref key.GetKeyPointerRef(isQuery ? 0 : psfOrdinal, this.KeyPointerSize); + ref KeyPointer keyPointer = ref key.GetKeyPointerRef(psfOrdinal, this.KeyPointerSize); return Utility.GetHashCode(this.userComparer.GetHashCode64(ref keyPointer.Key)) ^ Utility.GetHashCode(keyPointer.PsfOrdinal + 1); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool EqualsAtKeyAddress(ref CompositeKey queryKey, long physicalAddress) - { - // The query key only has a single key in it--the one we're trying to match. - ref KeyPointer queryKeyPointer = ref queryKey.GetKeyPointerRef(0, this.KeyPointerSize); - ref KeyPointer storedKeyPointer = ref GetKeyPointerRef(physicalAddress); - return KeysEqual(ref queryKeyPointer, ref storedKeyPointer); - } + public bool EqualsAtKeyAddress(ref KeyPointer queryKeyPointer, long physicalAddress) + => KeysEqual(ref queryKeyPointer, ref GetKeyPointerRef(physicalAddress)); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool EqualsAtRecordAddress(ref CompositeKey queryKey, long physicalAddress) - { - // The query key only has a single key in it--the one we're trying to match. - ref KeyPointer queryKeyPointer = ref queryKey.GetKeyPointerRef(0, this.KeyPointerSize); - ref KeyPointer storedKeyPointer = ref GetKeyPointerRef(physicalAddress, queryKeyPointer.PsfOrdinal); - return KeysEqual(ref queryKeyPointer, ref storedKeyPointer); - } + public bool EqualsAtRecordAddress(ref KeyPointer queryKeyPointer, long physicalAddress) + => KeysEqual(ref queryKeyPointer, ref GetKeyPointerRef(physicalAddress, queryKeyPointer.PsfOrdinal)); [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool KeysEqual(ref KeyPointer queryKeyPointer, ref KeyPointer storedKeyPointer) - { - return queryKeyPointer.PsfOrdinal == storedKeyPointer.PsfOrdinal && - this.userComparer.Equals(ref queryKeyPointer.Key, ref storedKeyPointer.Key); - } + => queryKeyPointer.PsfOrdinal == storedKeyPointer.PsfOrdinal && this.userComparer.Equals(ref queryKeyPointer.Key, ref storedKeyPointer.Key); [MethodImpl(MethodImplOptions.AggressiveInlining)] internal ref KeyPointer GetKeyPointerRef(ref CompositeKey key, int psfOrdinal) @@ -120,27 +82,27 @@ internal unsafe ref KeyPointer GetKeyPointerRef(long physicalAddress) internal unsafe ref KeyPointer GetKeyPointerRef(long physicalAddress, int psfOrdinal) => ref Unsafe.AsRef>((byte*)GetKeyAddressFromRecordPhysicalAddress(physicalAddress, psfOrdinal)); - public string GetString(ref CompositeKey key, int psfOrdinal = -1) + public string GetString(ref CompositeKey compositeKey, int psfOrdinal = -1) { - var sb = new StringBuilder("{"); - if (psfOrdinal == -1) { + var sb = new StringBuilder("{"); for (var ii = 0; ii < this.KeyCount; ++ii) { if (ii > 0) sb.Append(", "); - ref KeyPointer keyPointer = ref this.GetKeyPointerRef(ref key, ii); + ref KeyPointer keyPointer = ref this.GetKeyPointerRef(ref compositeKey, ii); sb.Append(keyPointer.IsNull ? "null" : keyPointer.Key.ToString()); } + sb.Append("}"); + return sb.ToString(); } - else - { - ref KeyPointer keyPointer = ref this.GetKeyPointerRef(ref key, psfOrdinal); - sb.Append(keyPointer.IsNull ? "null" : keyPointer.Key.ToString()); - } - sb.Append("}"); - return sb.ToString(); + return this.GetString(ref this.GetKeyPointerRef(ref compositeKey, psfOrdinal)); + } + + public string GetString(ref KeyPointer keyPointer) + { + return $"{{{(keyPointer.IsNull ? "null" : keyPointer.Key.ToString())}}}"; } } -} +} \ No newline at end of file diff --git a/cs/src/core/Index/PSF/KeyPointer.cs b/cs/src/core/Index/PSF/KeyPointer.cs index 90214a658..5d62cdeac 100644 --- a/cs/src/core/Index/PSF/KeyPointer.cs +++ b/cs/src/core/Index/PSF/KeyPointer.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. +using System.Runtime.CompilerServices; + namespace FASTER.core { internal struct KeyPointer @@ -13,24 +15,30 @@ internal struct KeyPointer /// /// The ordinal of the current . /// - internal ushort PsfOrdinal; + internal byte PsfOrdinal; // Note: consistent with Constants.kInvalidPsfOrdinal /// /// Flags regarding the PSF. /// - internal ushort Flags; + internal byte Flags; // TODO: Change to PSFResultFlags + + internal ushort Reserved; // TODOperf: Can be used for offset to start of key list, especially if we allow variable # of keys /// /// The Key returned by the execution. /// - internal TPSFKey Key; + internal TPSFKey Key; // TODOperf: for Key size > 4, reinterpret this an offset to the actual value (after the KeyPointer list) private const ushort NullFlag = 0x0001; internal bool IsNull { get => (this.Flags & NullFlag) == NullFlag; - set => this.Flags = (ushort)(value ? (this.Flags | NullFlag) : (this.Flags & ~NullFlag)); + set => this.Flags = (byte)(value ? (this.Flags | NullFlag) : (this.Flags & ~NullFlag)); } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal unsafe static ref KeyPointer CastFromKeyRef(ref TPSFKey keyRef) + => ref Unsafe.AsRef>((byte*)Unsafe.AsPointer(ref keyRef)); } } diff --git a/cs/src/core/Index/PSF/PSFContext.cs b/cs/src/core/Index/PSF/PSFContext.cs deleted file mode 100644 index cb4eae3a2..000000000 --- a/cs/src/core/Index/PSF/PSFContext.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace FASTER.core -{ - /// - /// Context for operations on the secondary FasterKV instance. - /// - public struct PSFContext - { - } -} diff --git a/cs/src/core/Index/PSF/PSFFunctions.cs b/cs/src/core/Index/PSF/PSFFunctions.cs index 4d584f59f..d364f22a4 100644 --- a/cs/src/core/Index/PSF/PSFFunctions.cs +++ b/cs/src/core/Index/PSF/PSFFunctions.cs @@ -12,53 +12,53 @@ namespace FASTER.core /// /// The type of the result key /// The type of the value - public class PSFFunctions : IFunctions, TRecordId, PSFInputSecondary, - PSFOutputSecondary, PSFContext> + public class PSFFunctions : IFunctions, + PSFOutputSecondary, Empty> where TPSFKey: struct where TRecordId: struct { // TODO: remove stuff that has been moved to PSFOutput.Visit, etc. #region Upserts - public bool ConcurrentWriter(ref CompositeKey _, ref TRecordId src, ref TRecordId dst) + public bool ConcurrentWriter(ref TPSFKey _, ref TRecordId src, ref TRecordId dst) { dst = src; return true; } - public void SingleWriter(ref CompositeKey _, ref TRecordId src, ref TRecordId dst) + public void SingleWriter(ref TPSFKey _, ref TRecordId src, ref TRecordId dst) => dst = src; - public void UpsertCompletionCallback(ref CompositeKey _, ref TRecordId value, PSFContext ctx) + public void UpsertCompletionCallback(ref TPSFKey _, ref TRecordId value, Empty ctx) { /* TODO: UpsertCompletionCallback */ } #endregion Upserts #region Reads - public void ConcurrentReader(ref CompositeKey key, ref PSFInputSecondary input, ref TRecordId value, ref PSFOutputSecondary dst) + public void ConcurrentReader(ref TPSFKey key, ref PSFInputSecondary input, ref TRecordId value, ref PSFOutputSecondary dst) => throw new PSFInternalErrorException("PSFOutput.Visit instead of ConcurrentReader should be called on PSF-implementing FasterKVs"); - public unsafe void SingleReader(ref CompositeKey _, ref PSFInputSecondary input, ref TRecordId value, ref PSFOutputSecondary dst) + public unsafe void SingleReader(ref TPSFKey _, ref PSFInputSecondary input, ref TRecordId value, ref PSFOutputSecondary dst) => throw new PSFInternalErrorException("PSFOutput.Visit instead of SingleReader should be called on PSF-implementing FasterKVs"); - public void ReadCompletionCallback(ref CompositeKey _, ref PSFInputSecondary input, ref PSFOutputSecondary output, PSFContext ctx, Status status) + public void ReadCompletionCallback(ref TPSFKey _, ref PSFInputSecondary input, ref PSFOutputSecondary output, Empty ctx, Status status) { /* TODO: ReadCompletionCallback */ } #endregion Reads #region RMWs - public void CopyUpdater(ref CompositeKey _, ref PSFInputSecondary input, ref TRecordId oldValue, ref TRecordId newValue) + public void CopyUpdater(ref TPSFKey _, ref PSFInputSecondary input, ref TRecordId oldValue, ref TRecordId newValue) => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); - public void InitialUpdater(ref CompositeKey _, ref PSFInputSecondary input, ref TRecordId value) + public void InitialUpdater(ref TPSFKey _, ref PSFInputSecondary input, ref TRecordId value) => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); - public bool InPlaceUpdater(ref CompositeKey _, ref PSFInputSecondary input, ref TRecordId value) + public bool InPlaceUpdater(ref TPSFKey _, ref PSFInputSecondary input, ref TRecordId value) => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); - public void RMWCompletionCallback(ref CompositeKey _, ref PSFInputSecondary input, PSFContext ctx, Status status) + public void RMWCompletionCallback(ref TPSFKey _, ref PSFInputSecondary input, Empty ctx, Status status) => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); #endregion RMWs - public void DeleteCompletionCallback(ref CompositeKey _, PSFContext ctx) + public void DeleteCompletionCallback(ref TPSFKey _, Empty ctx) { /* TODO: DeleteCompletionCallback */ } public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index 981a532cb..f4c33796e 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -25,7 +25,7 @@ public class PSFGroup : IExecutePSF, TRecordId> fht; + internal FasterKV fht; private readonly PSFFunctions functions; internal IPSFDefinition[] psfDefinitions; private readonly PSFRegistrationSettings regSettings; @@ -39,18 +39,18 @@ public class PSFGroup : IExecutePSF)); private readonly int recordIdSize = (Utility.GetSize(default(TRecordId)) + sizeof(long) - 1) & ~(sizeof(long) - 1); - internal ConcurrentStack, + internal ConcurrentStack, - PSFOutputSecondary, PSFContext, PSFFunctions>> freeSessions - = new ConcurrentStack, + PSFOutputSecondary, Empty, PSFFunctions>> freeSessions + = new ConcurrentStack, - PSFOutputSecondary, PSFContext, PSFFunctions>>(); - internal ConcurrentBag, + PSFOutputSecondary, Empty, PSFFunctions>>(); + internal ConcurrentBag, - PSFOutputSecondary, PSFContext, PSFFunctions>> allSessions - = new ConcurrentBag, + PSFOutputSecondary, Empty, PSFFunctions>> allSessions + = new ConcurrentBag, - PSFOutputSecondary, PSFContext, PSFFunctions>>(); + PSFOutputSecondary, Empty, PSFFunctions>>(); /// /// The list of s in this group @@ -94,10 +94,10 @@ public PSFGroup(PSFRegistrationSettings regSettings, IPSFDefinition(); - this.fht = new FasterKV, TRecordId>( + this.fht = new FasterKV( regSettings.HashTableSize, regSettings.LogSettings, this.checkpointSettings, null /*SerializerSettings*/, new CompositeKey.UnusedKeyComparer(), - new VariableLengthStructSettings, TRecordId> + new VariableLengthStructSettings { keyLength = new CompositeKey.VarLenLength(this.keyPointerSize, this.PSFCount) } @@ -108,22 +108,22 @@ public PSFGroup(PSFRegistrationSettings regSettings, IPSFDefinition, TRecordId, PSFInputSecondary, - PSFOutputSecondary, PSFContext, + private ClientSession, + PSFOutputSecondary, Empty, PSFFunctions> GetSession() { // Sessions are used only on post-RegisterPSF actions (Upsert, RMW, Query). if (this.freeSessions.TryPop(out var session)) return session; - session = this.fht.NewSession, PSFOutputSecondary, PSFContext, PSFFunctions>( + session = this.fht.NewSession, PSFOutputSecondary, Empty, PSFFunctions>( new PSFFunctions(), threadAffinitized: this.regSettings.ThreadAffinitized); this.allSessions.Add(session); return session; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ReleaseSession(ClientSession, TRecordId, PSFInputSecondary, - PSFOutputSecondary, PSFContext, + private void ReleaseSession(ClientSession, + PSFOutputSecondary, Empty, PSFFunctions> session) => this.freeSessions.Push(session); @@ -163,8 +163,8 @@ internal unsafe void MarkChanges(ref GroupKeysPair keysPair) { ref GroupKeys before = ref keysPair.Before; ref GroupKeys after = ref keysPair.After; - ref CompositeKey beforeCompKey = ref before.GetCompositeKeyRef>(); - ref CompositeKey afterCompKey = ref after.GetCompositeKeyRef>(); + ref CompositeKey beforeCompKey = ref before.CastToKeyRef>(); + ref CompositeKey afterCompKey = ref after.CastToKeyRef>(); for (var ii = 0; ii < this.PSFCount; ++ii) { var beforeIsNull = before.IsNullAt(ii); @@ -186,7 +186,6 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor { // Note: stackalloc is safe because PendingContext or PSFChangeTracker will copy it to the bufferPool // if needed. On the Insert fast path, we don't want any allocations otherwise; changeTracker is null. - // TODO: Cutoff (psfCount * keyPointerSize) per group to use bufferPool so stackalloc doesn't overflow. var keyMemLen = this.keyPointerSize * this.PSFCount; var keyBytes = stackalloc byte[keyMemLen]; @@ -198,7 +197,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor { ref KeyPointer keyPointer = ref Unsafe.AsRef>(keyBytes + ii * this.keyPointerSize); keyPointer.PrevAddress = Constants.kInvalidAddress; - keyPointer.PsfOrdinal = (ushort)ii; // TODO set limit in # of psfs to 2 << 15 + keyPointer.PsfOrdinal = (byte)ii; var key = this.psfDefinitions[ii].Execute(providerData); keyPointer.IsNull = !key.HasValue; @@ -214,7 +213,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor return Status.OK; ref CompositeKey compositeKey = ref Unsafe.AsRef>(keyBytes); - var input = new PSFInputSecondary(0, this.keyAccessor, this.Id, flags); + var input = new PSFInputSecondary(0, this.Id, flags); var value = recordId; int groupOrdinal = -1; @@ -256,10 +255,10 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor var lsn = session.ctx.serialNum + 1; return phase switch { - PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey, ref value, ref input, lsn), + PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey.CastToFirstKeyPointerRefAsKeyRef(), ref value, ref input, lsn), PSFExecutePhase.PostUpdate => session.PsfUpdate(ref changeTracker.GetGroupRef(groupOrdinal), ref value, ref input, lsn, changeTracker), - PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey, ref value, ref input, lsn, + PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey.CastToFirstKeyPointerRefAsKeyRef(), ref value, ref input, lsn, changeTracker), _ => throw new PSFInternalErrorException("Unknown PSF execution Phase {phase}") }; @@ -327,27 +326,27 @@ public Status Delete(PSFChangeTracker changeTracker) return this.ExecuteAndStore(changeTracker.BeforeData, default, PSFExecutePhase.Delete, changeTracker); } - /// - public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe PSFInputSecondary MakeQueryInput(int psfOrdinal, ref TPSFKey key) { // Putting the query key in PSFInput is necessary because iterator functions cannot contain unsafe code or have // byref args, and bufferPool is needed here because the stack goes away as part of the iterator operation. - // TODOperf: PSFInput* and PSFOutput* are classes because we communicate through interfaces to avoid - // having to add additional generic args. Interfaces on structs incur boxing overhead (plus the complexity - // of mutable structs). But check the performance here; if necessary perhaps I can figure out a way to - // pass a struct with no TPSFKey, TRecordId, etc. and use an FHT-level interface to manage it (async too). - var psfInput = new PSFInputSecondary(psfOrdinal, this.keyAccessor, this.Id); - psfInput.SetQueryKey(this.bufferPool, ref key); - return Query(psfInput, querySettings); + var psfInput = new PSFInputSecondary(psfOrdinal, this.Id); + psfInput.SetQueryKey(this.bufferPool, this.keyAccessor, ref key); + return psfInput; } + /// + public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings) + => Query(MakeQueryInput(psfOrdinal, ref key), querySettings); + private IEnumerable Query(PSFInputSecondary input, PSFQuerySettings querySettings) { // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr. Also consider // having TPSFKeys[] for a single PSF walk through in parallel, so the FHT log memory access is sequential. var secondaryOutput = new PSFOutputSecondary(this.keyAccessor); - var readArgs = new PSFReadArgs, TRecordId>(input, secondaryOutput); + var readArgs = new PSFReadArgs(input, secondaryOutput); var session = this.GetSession(); var deadRecs = new DeadRecords(); @@ -395,22 +394,15 @@ private IEnumerable Query(PSFInputSecondary input, PSFQueryS #if DOTNETCORE /// public unsafe IAsyncEnumerable QueryAsync(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings) - { - // See comments on non-async version. - var psfInput = new PSFInputSecondary(psfOrdinal, this.keyAccessor, this.Id); - psfInput.SetQueryKey(this.bufferPool, ref key); - return QueryAsync(psfInput, querySettings); - } + => QueryAsync(MakeQueryInput(psfOrdinal, ref key), querySettings); private async IAsyncEnumerable QueryAsync(PSFInputSecondary input, PSFQuerySettings querySettings) { - // TODO: Improved enumerator in DurTask - // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr. Also consider // having TPSFKeys[] for a single PSF walk through in parallel, so the FHT log memory access is sequential. var secondaryOutput = new PSFOutputSecondary(this.keyAccessor); - var readArgs = new PSFReadArgs, TRecordId>(input, secondaryOutput); + var readArgs = new PSFReadArgs(input, secondaryOutput); var session = this.GetSession(); var deadRecs = new DeadRecords(); diff --git a/cs/src/core/Index/PSF/PSFInput.cs b/cs/src/core/Index/PSF/PSFInput.cs index d9956d120..4c75703ba 100644 --- a/cs/src/core/Index/PSF/PSFInput.cs +++ b/cs/src/core/Index/PSF/PSFInput.cs @@ -48,8 +48,6 @@ public interface IPSFInput ref TKey QueryKeyRef { get; } } - // TODO: Trim IPSFInput to only what PSFInputPrimaryReadAddress needs - /// /// Input to PsfInternalReadAddress on the primary (stores user values) FasterKV to retrieve the Key and Value /// for a logicalAddress returned from the secondary FasterKV instances. This class is FasterKV-provider-specific. @@ -92,26 +90,24 @@ public bool IsDelete /// Input to operations on the secondary FasterKV instance (stores PSF chains) for everything /// except reading based on a LogicalAddress. /// - public unsafe class PSFInputSecondary : IPSFInput>, IDisposable + public unsafe class PSFInputSecondary : IPSFInput, IDisposable where TPSFKey : struct { - internal readonly KeyAccessor keyAccessor; - internal PSFResultFlags* resultFlags; + internal PSFResultFlags* resultFlags; // TODO: Replace with KeyPointer.Flags; remember to clear update bits private SectorAlignedMemory keyPointerMem; - internal PSFInputSecondary(int psfOrdinal, KeyAccessor keyAcc, long groupId, PSFResultFlags* flags = null) + internal PSFInputSecondary(int psfOrdinal, long groupId, PSFResultFlags* flags = null) { this.PsfOrdinal = psfOrdinal; - this.keyAccessor = keyAcc; this.GroupId = groupId; this.resultFlags = flags; this.ReadLogicalAddress = Constants.kInvalidAddress; } - internal void SetQueryKey(SectorAlignedBufferPool pool, ref TPSFKey key) + internal void SetQueryKey(SectorAlignedBufferPool pool, KeyAccessor keyAccessor, ref TPSFKey key) { // Create a varlen CompositeKey with just one item. This is ONLY used as the query key to QueryPSF. - this.keyPointerMem = pool.Get(this.keyAccessor.KeyPointerSize); + this.keyPointerMem = pool.Get(keyAccessor.KeyPointerSize); ref KeyPointer keyPointer = ref Unsafe.AsRef>(keyPointerMem.GetValidPointer()); keyPointer.PrevAddress = Constants.kInvalidAddress; keyPointer.PsfOrdinal = (ushort)this.PsfOrdinal; @@ -130,8 +126,8 @@ internal void SetQueryKey(SectorAlignedBufferPool pool, ref TPSFKey key) public long ReadLogicalAddress { get; set; } - public ref CompositeKey QueryKeyRef - => ref Unsafe.AsRef>(this.keyPointerMem.GetValidPointer()); + public ref TPSFKey QueryKeyRef + => ref Unsafe.AsRef(this.keyPointerMem.GetValidPointer()); public void Dispose() { diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index 366441726..b7a0ebea6 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; // TODO: Remove PackageId and PackageOutputPath from csproj when this is folded into master +// TODO: Make a new FASTER.PSF.dll namespace FASTER.core { @@ -205,6 +206,13 @@ internal IPSF[] RegisterPSF(PSFRegistrationSettings registrati if (defs is null || defs.Length == 0 || defs.Any(def => def is null) || defs.Length == 0) throw new PSFArgumentException("PSF definitions cannot be null or empty"); + // We use stackalloc for speed and can recurse in pending operations, so make sure we don't blow the stack. + if (defs.Length > Constants.kInvalidPsfOrdinal) + throw new PSFArgumentException($"There can be no more than {Constants.kInvalidPsfOrdinal} PSFs in a single Group"); + const int maxKeySize = 256; + if (Utility.GetSize(default(KeyPointer)) > maxKeySize) + throw new PSFArgumentException($"The size of the PSF key can be no more than {maxKeySize} bytes"); + // This is a very rare operation and unlikely to have any contention, and locking the dictionary // makes it much easier to recover from duplicates if needed. lock (this.psfNames) diff --git a/cs/src/core/Index/PSF/PSFOutput.cs b/cs/src/core/Index/PSF/PSFOutput.cs index afaee45e3..ea305e755 100644 --- a/cs/src/core/Index/PSF/PSFOutput.cs +++ b/cs/src/core/Index/PSF/PSFOutput.cs @@ -14,7 +14,7 @@ namespace FASTER.core /// /// The type of the Key, either a for the /// secondary FasterKV instances, or the user's TKVKey for the primary FasterKV instance. - /// The type of the Key, either a TRecordId for the + /// The type of the Value, either a TRecordId for the /// secondary FasterKV instances, or the user's TKVValue for the primary FasterKV instance. public interface IPSFOutput { @@ -61,11 +61,11 @@ public PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, ref TKVVal /// /// The type of the key returned from a /// The type of the provider's record identifier - public unsafe class PSFOutputSecondary : IPSFOutput, TRecordId> + public unsafe class PSFOutputSecondary : IPSFOutput where TPSFKey : struct where TRecordId : struct { - private KeyAccessor keyAccessor; + private readonly KeyAccessor keyAccessor; internal TRecordId RecordId { get; private set; } @@ -81,13 +81,14 @@ internal PSFOutputSecondary(KeyAccessor keyAcc) this.PreviousLogicalAddress = Constants.kInvalidAddress; } - public PSFOperationStatus Visit(int psfOrdinal, ref CompositeKey key, + public PSFOperationStatus Visit(int psfOrdinal, ref TPSFKey key, ref TRecordId value, bool tombstone, bool isConcurrent) { // This is the secondary FKV; we hold onto the RecordId and create the provider data when QueryPSF returns. this.RecordId = value; this.Tombstone = tombstone; - ref KeyPointer keyPointer = ref this.keyAccessor.GetKeyPointerRef(ref key, psfOrdinal); + ref CompositeKey compositeKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref key); + ref KeyPointer keyPointer = ref this.keyAccessor.GetKeyPointerRef(ref compositeKey, psfOrdinal); Debug.Assert(keyPointer.PsfOrdinal == (ushort)psfOrdinal, "Visit found mismatched PSF ordinal"); this.PreviousLogicalAddress = keyPointer.PrevAddress; return new PSFOperationStatus(OperationStatus.SUCCESS); diff --git a/cs/src/core/Index/PSF/PSFResultFlags.cs b/cs/src/core/Index/PSF/PSFResultFlags.cs index b59998d60..719e6dc73 100644 --- a/cs/src/core/Index/PSF/PSFResultFlags.cs +++ b/cs/src/core/Index/PSF/PSFResultFlags.cs @@ -12,7 +12,7 @@ namespace FASTER.core public enum PSFResultFlags { /// - /// For initialization only + /// No flags set /// None = 0, diff --git a/cs/src/core/Index/PSF/RecordIterator.cs b/cs/src/core/Index/PSF/RecordIterator.cs index 7c7dd0e0a..4191ff540 100644 --- a/cs/src/core/Index/PSF/RecordIterator.cs +++ b/cs/src/core/Index/PSF/RecordIterator.cs @@ -174,7 +174,7 @@ internal KeyTypeRecordIterator(int keyTypeOrd, IEnumerable<(IPSF psf, IEnumerato #endregion Sync methods } -#if DOTNETCORE // TODO: Refactor some of this cut and paste to a KeyTypeRecordIteratorBase templated on the record enumerable type +#if DOTNETCORE /// /// A single TPSFKey type's vector of its PSFs' async streams of recordIds (each TPSFKey type may have multiple PSFs being queried). /// From 77501414399bfa7aac86275ca0ccb7e3c6d979b1 Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Mon, 24 Aug 2020 20:44:32 -0700 Subject: [PATCH 15/19] Move PSFResultFlags into KeyPointer; add OffsetToStartOfKeys to make it easier in future to omit IsNull keys from the composite key --- .../FasterPSFSample/FasterPSFSample.cs | 5 +- cs/src/core/Index/FASTER/FASTERThread.cs | 2 +- cs/src/core/Index/PSF/CompositeKey.cs | 7 ++ .../Index/PSF/FasterPSFContextOperations.cs | 6 +- cs/src/core/Index/PSF/FasterPSFImpl.cs | 29 ++++--- .../Index/PSF/FasterPSFSessionOperations.cs | 2 +- cs/src/core/Index/PSF/GroupCompositeKey.cs | 56 +++++++++++++ cs/src/core/Index/PSF/GroupKeys.cs | 72 ----------------- cs/src/core/Index/PSF/KeyAccessor.cs | 41 +++++++--- cs/src/core/Index/PSF/KeyPointer.cs | 80 ++++++++++++++++--- cs/src/core/Index/PSF/PSFChangeTracker.cs | 12 +-- cs/src/core/Index/PSF/PSFGroup.cs | 62 +++++++------- cs/src/core/Index/PSF/PSFInput.cs | 28 +------ cs/src/core/Index/PSF/PSFOutput.cs | 4 +- cs/src/core/Index/PSF/PSFResultFlags.cs | 36 --------- 15 files changed, 230 insertions(+), 212 deletions(-) create mode 100644 cs/src/core/Index/PSF/GroupCompositeKey.cs delete mode 100644 cs/src/core/Index/PSF/GroupKeys.cs delete mode 100644 cs/src/core/Index/PSF/PSFResultFlags.cs diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs index 5a8500eea..8cc414245 100644 --- a/cs/playground/FasterPSFSample/FasterPSFSample.cs +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -68,13 +68,12 @@ internal async static Task RunSample>> "); - Console.WriteLine(ok ? "Passed! All operations succeeded" - : "*** Failed! *** One or more operations did not succeed"); + Console.WriteLine(ok ? "Passed! All operations succeeded" : "*** Failed! *** One or more operations failed"); Console.WriteLine(); if (Debugger.IsAttached) { Console.Write("Press ENTER to close this window . . ."); - Console.ReadKey(); + Console.ReadLine(); } } finally diff --git a/cs/src/core/Index/FASTER/FASTERThread.cs b/cs/src/core/Index/FASTER/FASTERThread.cs index 16d023433..c699392ef 100644 --- a/cs/src/core/Index/FASTER/FASTERThread.cs +++ b/cs/src/core/Index/FASTER/FASTERThread.cs @@ -270,7 +270,7 @@ internal void InternalCompleteRetryRequest); PsfRcuInsert(groupKeysPair.After, ref value, ref pendingContext.input, diff --git a/cs/src/core/Index/PSF/CompositeKey.cs b/cs/src/core/Index/PSF/CompositeKey.cs index d7f8920ca..bc486ff16 100644 --- a/cs/src/core/Index/PSF/CompositeKey.cs +++ b/cs/src/core/Index/PSF/CompositeKey.cs @@ -53,6 +53,13 @@ internal static ref CompositeKey CastFromFirstKeyPointerRefAsKeyRef(ref internal ref TPSFKey CastToFirstKeyPointerRefAsKeyRef() => ref Unsafe.AsRef((byte*)Unsafe.AsPointer(ref this)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ClearUpdateFlags(int psfCount, int keyPointerSize) + { + for (var ii = 0; ii < psfCount; ++ii) + this.GetKeyPointerRef(0, keyPointerSize).ClearUpdateFlags(); + } + internal class VarLenLength : IVariableLengthStruct { private readonly int size; diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs index 3ef3fb173..17af1782d 100644 --- a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -124,7 +124,7 @@ internal Status ContextPsfInsert(ref Key } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfUpdate(ref GroupKeysPair groupKeysPair, ref Value value, ref Input input, + internal Status ContextPsfUpdate(ref GroupCompositeKeyPair groupKeysPair, ref Value value, ref Input input, FasterSession fasterSession, long serialNo, FasterExecutionContext sessionCtx, PSFChangeTracker changeTracker) @@ -134,7 +134,6 @@ internal Status ContextPsfUpdate)input; var groupKeys = groupKeysPair.Before; - unsafe { psfInput.SetFlags(groupKeys.ResultFlags); } psfInput.IsDelete = true; var internalStatus = this.PsfInternalInsert(ref groupKeys.CastToKeyRef(), ref value, ref input, @@ -153,13 +152,12 @@ internal Status ContextPsfUpdate(GroupKeys groupKeys, ref Value value, ref Input input, + private Status PsfRcuInsert(GroupCompositeKey groupKeys, ref Value value, ref Input input, ref PendingContext pcontext, FasterSession fasterSession, FasterExecutionContext sessionCtx, long serialNo) where FasterSession : IFasterSession { var psfInput = (IPSFInput)input; - unsafe { psfInput.SetFlags(groupKeys.ResultFlags); } psfInput.IsDelete = false; var internalStatus = this.PsfInternalInsert(ref groupKeys.CastToKeyRef(), ref value, ref input, ref pcontext, fasterSession, sessionCtx, serialNo); diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index 4cda9e5c7..856530c5e 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -36,7 +36,7 @@ bool ScanQueryChain(ref long logicalAddress, ref KeyPointer queryKeyPointer PsfTrace($" / {logicalAddress}"); return true; } - logicalAddress = this.PsfKeyAccessor.GetPrevAddress(physicalAddress); + logicalAddress = this.PsfKeyAccessor.GetPreviousAddress(physicalAddress); if (logicalAddress < hlog.HeadAddress) break; // RECORD_ON_DISK or not found physicalAddress = hlog.GetPhysicalAddress(logicalAddress); @@ -200,7 +200,7 @@ ref hlog.GetValue(recordAddress), pendingContext.output = default; pendingContext.userContext = default; pendingContext.entry.word = entry.word; - pendingContext.logicalAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyLogicalAddress(logicalAddress, psfInput.PsfOrdinal); + pendingContext.logicalAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(hlog.GetPhysicalAddress(logicalAddress)); pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; pendingContext.heldLatch = heldOperation; @@ -311,7 +311,7 @@ ref hlog.GetValue(recordAddress), pendingContext.userContext = default; pendingContext.entry.word = default; pendingContext.logicalAddress = this.ImplmentsPSFs - ? this.PsfKeyAccessor.GetRecordAddressFromKeyLogicalAddress(logicalAddress, psfInput.PsfOrdinal) + ? this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(hlog.GetPhysicalAddress(logicalAddress)) : logicalAddress; pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; @@ -329,6 +329,7 @@ unsafe struct CASHelper internal HashBucketEntry entry; internal long hash; internal int slot; + internal bool isNull; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -351,20 +352,25 @@ internal OperationStatus PsfInternalInsert> Constants.kHashTagShift); @@ -393,14 +399,14 @@ internal OperationStatus PsfInternalInsert> Constants.kHashTagShift); PsfTrace($" ({psfInput.PsfOrdinal}): {casHelper.hash} {tag} | newLA {newLogicalAddress} | prev {casHelper.entry.word}"); - if (psfInput.IsNullAt) + if (casHelper.isNull) { PsfTraceLine(" null"); continue; @@ -475,7 +481,7 @@ internal OperationStatus PsfInternalInsert.ReadAsyncResult(ref GroupKeysPair groupKeysPair, ref Value value, ref Input input, long serialNo, + internal Status PsfUpdate(ref GroupCompositeKeyPair groupKeysPair, ref Value value, ref Input input, long serialNo, PSFChangeTracker changeTracker) { // Called on the secondary FasterKV diff --git a/cs/src/core/Index/PSF/GroupCompositeKey.cs b/cs/src/core/Index/PSF/GroupCompositeKey.cs new file mode 100644 index 000000000..630b51013 --- /dev/null +++ b/cs/src/core/Index/PSF/GroupCompositeKey.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + internal struct GroupCompositeKey : IDisposable + { + // This cannot be typed to a TPSFKey because there may be different TPSFKeys across groups. + private SectorAlignedMemory KeyPointerMem; + + internal void Set(SectorAlignedMemory keyMem) => this.KeyPointerMem = keyMem; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal unsafe ref TCompositeOrIndividualKey CastToKeyRef() + => ref Unsafe.AsRef(this.KeyPointerMem.GetValidPointer()); + + public void Dispose() => this.KeyPointerMem?.Return(); + } + + internal struct GroupCompositeKeyPair : IDisposable + { + internal long GroupId; + + // If the PSFGroup found the RecordId in its IPUCache, we carry it here. + internal long LogicalAddress; + + internal GroupCompositeKey Before; + internal GroupCompositeKey After; + + internal GroupCompositeKeyPair(long id) + { + this.GroupId = id; + this.LogicalAddress = Constants.kInvalidAddress; + this.Before = default; + this.After = default; + this.HasChanges = false; + } + + internal bool HasAddress => this.LogicalAddress != Constants.kInvalidAddress; + + internal ref TCompositeKey GetBeforeKey() => ref this.Before.CastToKeyRef(); + + internal ref TCompositeKey GetAfterKey() => ref this.After.CastToKeyRef(); + + internal bool HasChanges; + + public void Dispose() + { + this.Before.Dispose(); + this.After.Dispose(); + } + } +} diff --git a/cs/src/core/Index/PSF/GroupKeys.cs b/cs/src/core/Index/PSF/GroupKeys.cs deleted file mode 100644 index 9654ef8bc..000000000 --- a/cs/src/core/Index/PSF/GroupKeys.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -using System; -using System.Runtime.CompilerServices; - -namespace FASTER.core -{ - internal struct GroupKeys : IDisposable - { - // This cannot be typed to a TPSFKey because there may be different TPSFKeys across groups. - private SectorAlignedMemory compositeKeyMem; - private SectorAlignedMemory flagsMem; - - internal void Set(SectorAlignedMemory keyMem, SectorAlignedMemory flagsMem) - { - this.compositeKeyMem = keyMem; - this.flagsMem = flagsMem; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal unsafe ref TCompositeOrIndividualKey CastToKeyRef() - => ref Unsafe.AsRef(this.compositeKeyMem.GetValidPointer()); - - internal unsafe PSFResultFlags* ResultFlags => (PSFResultFlags*)this.flagsMem.GetValidPointer(); - - internal unsafe bool IsNullAt(int psfOrdinal) => (this.ResultFlags + psfOrdinal)->HasFlag(PSFResultFlags.IsNull); - public unsafe bool IsUnlinkOldAt(int psfOrdinal) => (this.ResultFlags + psfOrdinal)->HasFlag(PSFResultFlags.UnlinkOld); - public unsafe bool IsLinkNewAt(int psfOrdinal) => (this.ResultFlags + psfOrdinal)->HasFlag(PSFResultFlags.LinkNew); - - public unsafe bool HasChanges => this.ResultFlags->HasFlag(PSFResultFlags.UnlinkOld) || this.ResultFlags->HasFlag(PSFResultFlags.LinkNew); - - public void Dispose() - { - this.compositeKeyMem?.Return(); - this.flagsMem?.Return(); - } - } - - internal struct GroupKeysPair : IDisposable - { - internal long GroupId; - - // If the PSFGroup found the RecordId in its IPUCache, we carry it here. - internal long LogicalAddress; - - internal GroupKeys Before; - internal GroupKeys After; - - internal GroupKeysPair(long id) - { - this.GroupId = id; - this.LogicalAddress = Constants.kInvalidAddress; - this.Before = default; - this.After = default; - } - - internal bool HasAddress => this.LogicalAddress != Constants.kInvalidAddress; - - internal ref TCompositeKey GetBeforeKey() => ref this.Before.CastToKeyRef(); - - internal ref TCompositeKey GetAfterKey() => ref this.After.CastToKeyRef(); - - internal bool HasChanges => this.After.HasChanges; - - public void Dispose() - { - this.Before.Dispose(); - this.After.Dispose(); - } - } -} diff --git a/cs/src/core/Index/PSF/KeyAccessor.cs b/cs/src/core/Index/PSF/KeyAccessor.cs index 0f4e2749c..24fb13629 100644 --- a/cs/src/core/Index/PSF/KeyAccessor.cs +++ b/cs/src/core/Index/PSF/KeyAccessor.cs @@ -27,25 +27,27 @@ internal KeyAccessor(IFasterEqualityComparer userComparer, int keyCount public int KeyPointerSize { get; } + #region KeyPointer accessors [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetPrevAddress(long physicalAddress) - => GetKeyPointerRef(physicalAddress).PrevAddress; + public long GetPreviousAddress(long physicalAddress) + => this.GetKeyPointerRef(physicalAddress).PreviousAddress; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetPrevAddress(ref CompositeKey key, int psfOrdinal, long prevAddress) - => this.GetKeyPointerRef(ref key, psfOrdinal).PrevAddress = prevAddress; + public void SetPreviousAddress(ref CompositeKey key, int psfOrdinal, long prevAddress) + => this.GetKeyPointerRef(ref key, psfOrdinal).PreviousAddress = prevAddress; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetRecordAddressFromKeyPhysicalAddress(long physicalAddress) - => physicalAddress - this.GetKeyPointerRef(physicalAddress).PsfOrdinal * this.KeyPointerSize - RecordInfo.GetLength(); // Note: Assumes all PSFs are present + public void SetOffsetToStartOfKeys(ref CompositeKey key, int psfOrdinal, int offset) + => this.GetKeyPointerRef(ref key, psfOrdinal).OffsetToStartOfKeys = offset; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetRecordAddressFromKeyLogicalAddress(long logicalAddress, int psfOrdinal) - => logicalAddress - psfOrdinal * this.KeyPointerSize - RecordInfo.GetLength(); // Note: Assumes all PSFs are present + public bool IsNullAt(ref CompositeKey key, int psfOrdinal) => this.GetKeyPointerRef(ref key, psfOrdinal).IsNull; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetKeyAddressFromRecordPhysicalAddress(long physicalAddress, int psfOrdinal) - => physicalAddress + RecordInfo.GetLength() + psfOrdinal * this.KeyPointerSize; // Note: Assumes all PSFs are present + public bool IsUnlinkOldAt(ref CompositeKey key, int psfOrdinal) => this.GetKeyPointerRef(ref key, psfOrdinal).IsUnlinkOld; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsLinkNewAt(ref CompositeKey key, int psfOrdinal) => this.GetKeyPointerRef(ref key, psfOrdinal).IsLinkNew; [MethodImpl(MethodImplOptions.AggressiveInlining)] public long GetHashCode64(ref KeyPointer keyPointer) @@ -81,7 +83,21 @@ internal unsafe ref KeyPointer GetKeyPointerRef(long physicalAddress) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal unsafe ref KeyPointer GetKeyPointerRef(long physicalAddress, int psfOrdinal) => ref Unsafe.AsRef>((byte*)GetKeyAddressFromRecordPhysicalAddress(physicalAddress, psfOrdinal)); + #endregion KeyPointer accessors + + #region Address manipulation + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetRecordAddressFromKeyPhysicalAddress(long physicalAddress) + => physicalAddress - this.GetKeyPointerRef(physicalAddress).OffsetToStartOfKeys - RecordInfo.GetLength(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetKeyAddressFromRecordPhysicalAddress(long physicalAddress, int psfOrdinal) + // TODOperf: if we omit IsNull keys, then this will have to walk to the key with psfOrdinal. Fortunately it is only + // called during AsyncGetFromDiskCallback. + => physicalAddress + RecordInfo.GetLength() + psfOrdinal * this.KeyPointerSize; + #endregion Address manipulation +#if DEBUG public string GetString(ref CompositeKey compositeKey, int psfOrdinal = -1) { if (psfOrdinal == -1) @@ -101,8 +117,7 @@ public string GetString(ref CompositeKey compositeKey, int psfOrdinal = } public string GetString(ref KeyPointer keyPointer) - { - return $"{{{(keyPointer.IsNull ? "null" : keyPointer.Key.ToString())}}}"; - } + => $"{{{(keyPointer.IsNull ? "null" : keyPointer.Key.ToString())}}}"; +#endif } } \ No newline at end of file diff --git a/cs/src/core/Index/PSF/KeyPointer.cs b/cs/src/core/Index/PSF/KeyPointer.cs index 5d62cdeac..0c6c575fe 100644 --- a/cs/src/core/Index/PSF/KeyPointer.cs +++ b/cs/src/core/Index/PSF/KeyPointer.cs @@ -7,36 +7,98 @@ namespace FASTER.core { internal struct KeyPointer { + #region Fields /// /// The previous address in the hash chain. May be for a different PsfOrdinal than this one due to hash collisions. /// - internal long PrevAddress; + internal long PreviousAddress; + + /// + /// The offset to the start of the record + /// + private ushort offsetToStartOfKeys; /// /// The ordinal of the current . /// - internal byte PsfOrdinal; // Note: consistent with Constants.kInvalidPsfOrdinal + private byte psfOrdinal; // Note: 'byte' is consistent with Constants.kInvalidPsfOrdinal /// /// Flags regarding the PSF. /// - internal byte Flags; // TODO: Change to PSFResultFlags - - internal ushort Reserved; // TODOperf: Can be used for offset to start of key list, especially if we allow variable # of keys + private byte flags; /// /// The Key returned by the execution. /// - internal TPSFKey Key; // TODOperf: for Key size > 4, reinterpret this an offset to the actual value (after the KeyPointer list) + internal TPSFKey Key; // TODOperf: for Key size > 4, reinterpret this an offset to the actual value (after the KeyPointer list) + #endregion Fields + + internal void Initialize(int psfOrdinal, ref TPSFKey key) + { + this.PreviousAddress = Constants.kInvalidAddress; + this.offsetToStartOfKeys = 0; + this.PsfOrdinal = psfOrdinal; + this.flags = 0; + this.Key = key; + } - private const ushort NullFlag = 0x0001; + #region Accessors + // For Insert, this identifies a null PSF result (the record does not match the PSF and is not included + // in any TPSFKey chain for it). Also used in PSFChangeTracker to determine whether to set kUnlinkOldBit. + private const byte kIsNullBit = 0x01; + + // If Key size is > 4, then reinterpret the Key as an offset to the actual key. (TODOperf not implemented) + private const byte kIsOutOfLineKeyBit = 0x02; + + // For Update, the TPSFKey has changed; remove this record from the previous TPSFKey chain. + private const byte kUnlinkOldBit = 0x04; + + // For Update and insert, the TPSFKey has changed; add this record to the new TPSFKey chain. + private const byte kLinkNewBit = 0x08; internal bool IsNull { - get => (this.Flags & NullFlag) == NullFlag; - set => this.Flags = (byte)(value ? (this.Flags | NullFlag) : (this.Flags & ~NullFlag)); + get => (this.flags & kIsNullBit) != 0; + set => this.flags = value ? (byte)(this.flags | kIsNullBit) : (byte)(this.flags & ~kIsNullBit); + } + + internal bool IsUnlinkOld + { + get => (this.flags & kUnlinkOldBit) != 0; + set => this.flags = value ? (byte)(this.flags | kUnlinkOldBit) : (byte)(this.flags & ~kUnlinkOldBit); + } + + internal bool IsLinkNew + { + get => (this.flags & kLinkNewBit) != 0; + set => this.flags = value ? (byte)(this.flags | kLinkNewBit) : (byte)(this.flags & ~kLinkNewBit); + } + + internal bool IsOutOfLineKey + { + get => (this.flags & kIsOutOfLineKeyBit) != 0; + set => this.flags = value ? (byte)(this.flags | kIsOutOfLineKeyBit) : (byte)(this.flags & ~kIsOutOfLineKeyBit); + } + + internal bool HasChanges => (this.flags & (kUnlinkOldBit | kLinkNewBit)) != 0; + + internal int PsfOrdinal + { + get => this.psfOrdinal; + set => this.psfOrdinal = (byte)value; } + internal int OffsetToStartOfKeys + { + get => this.offsetToStartOfKeys; + set => this.offsetToStartOfKeys = (ushort)value; + } + #endregion Accessors + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ClearUpdateFlags() => this.flags = (byte)(this.flags & ~(kUnlinkOldBit | kLinkNewBit)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal unsafe static ref KeyPointer CastFromKeyRef(ref TPSFKey keyRef) => ref Unsafe.AsRef>((byte*)Unsafe.AsPointer(ref keyRef)); diff --git a/cs/src/core/Index/PSF/PSFChangeTracker.cs b/cs/src/core/Index/PSF/PSFChangeTracker.cs index aa141bdee..5b6d5d886 100644 --- a/cs/src/core/Index/PSF/PSFChangeTracker.cs +++ b/cs/src/core/Index/PSF/PSFChangeTracker.cs @@ -43,14 +43,14 @@ internal void SetAfterData(TProviderData data, TRecordId recordId) public UpdateOperation UpdateOp { get; set; } #endregion Data API - private GroupKeysPair[] groups; + private GroupCompositeKeyPair[] groups; internal bool HasBeforeKeys { get; set; } internal long CachedBeforeLA = Constants.kInvalidAddress; internal PSFChangeTracker(IEnumerable groupIds) { - this.groups = groupIds.Select(id => new GroupKeysPair(id)).ToArray(); + this.groups = groupIds.Select(id => new GroupCompositeKeyPair(id)).ToArray(); } internal bool FindGroup(long groupId, out int ordinal) @@ -69,19 +69,19 @@ internal bool FindGroup(long groupId, out int ordinal) return false; } - internal ref GroupKeysPair GetGroupRef(int ordinal) => ref groups[ordinal]; + internal ref GroupCompositeKeyPair GetGroupRef(int ordinal) => ref groups[ordinal]; - internal ref GroupKeysPair FindGroupRef(long groupId, long logAddr = Constants.kInvalidAddress) + internal ref GroupCompositeKeyPair FindGroupRef(long groupId, long logAddr = Constants.kInvalidAddress) { if (!this.FindGroup(groupId, out var ordinal)) { // A new group was added while we were populating this changeTracker; should be quite rare. // TODOtest: this case - var groups = new GroupKeysPair[this.groups.Length + 1]; + var groups = new GroupCompositeKeyPair[this.groups.Length + 1]; Array.Copy(this.groups, groups, this.groups.Length); this.groups = groups; ordinal = this.groups.Length - 1; } - ref GroupKeysPair ret = ref this.groups[ordinal]; + ref GroupCompositeKeyPair ret = ref this.groups[ordinal]; ret.GroupId = groupId; ret.LogicalAddress = logAddr; return ref ret; diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index f4c33796e..51fe14fc1 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -150,33 +150,41 @@ public PSF this[string name] => Array.Find(this.PSFs, psf => psf.Name.Equals(name, StringComparison.CurrentCultureIgnoreCase)) ?? throw new PSFArgumentException("PSF not found"); - private unsafe void StoreKeys(ref GroupKeys keys, byte* kPtr, int kLen, PSFResultFlags* flagsPtr, int flagsLen) + private unsafe void StoreKeys(ref GroupCompositeKey keys, byte* keysPtr, int keysLen) { - var poolKeyMem = this.bufferPool.Get(kLen); - Buffer.MemoryCopy(kPtr, poolKeyMem.GetValidPointer(), kLen, kLen); - var flagsMem = this.bufferPool.Get(flagsLen); - Buffer.MemoryCopy(flagsPtr, flagsMem.GetValidPointer(), flagsLen, flagsLen); - keys.Set(poolKeyMem, flagsMem); + var poolKeyMem = this.bufferPool.Get(keysLen); + Buffer.MemoryCopy(keysPtr, poolKeyMem.GetValidPointer(), keysLen, keysLen); + keys.Set(poolKeyMem); } - internal unsafe void MarkChanges(ref GroupKeysPair keysPair) + internal unsafe void MarkChanges(ref GroupCompositeKeyPair keysPair) { - ref GroupKeys before = ref keysPair.Before; - ref GroupKeys after = ref keysPair.After; - ref CompositeKey beforeCompKey = ref before.CastToKeyRef>(); - ref CompositeKey afterCompKey = ref after.CastToKeyRef>(); + ref GroupCompositeKey before = ref keysPair.Before; + ref GroupCompositeKey after = ref keysPair.After; + ref CompositeKey beforeCompositeKey = ref before.CastToKeyRef>(); + ref CompositeKey afterCompositeKey = ref after.CastToKeyRef>(); for (var ii = 0; ii < this.PSFCount; ++ii) { - var beforeIsNull = before.IsNullAt(ii); - var afterIsNull = after.IsNullAt(ii); - var keysEqual = !beforeIsNull && !afterIsNull - && beforeCompKey.GetKeyRef(ii, this.keyPointerSize).Equals(afterCompKey.GetKeyRef(ii, this.keyPointerSize)); + ref KeyPointer beforeKeyPointer = ref beforeCompositeKey.GetKeyPointerRef(ii, this.keyPointerSize); + ref KeyPointer afterKeyPointer = ref afterCompositeKey.GetKeyPointerRef(ii, this.keyPointerSize); + + var beforeIsNull = beforeKeyPointer.IsNull; + var afterIsNull = afterKeyPointer.IsNull; + var keysEqual = !beforeIsNull && !afterIsNull && beforeKeyPointer.Key.Equals(afterKeyPointer.Key); // IsNull is already set in PSFGroup.ExecuteAndStore. - if (!before.IsNullAt(ii) && (after.IsNullAt(ii) || !keysEqual)) - *after.ResultFlags |= PSFResultFlags.UnlinkOld; - if (!after.IsNullAt(ii) && (before.IsNullAt(ii) || !keysEqual)) - *after.ResultFlags |= PSFResultFlags.LinkNew; + if ((beforeIsNull != afterIsNull) || !keysEqual) + keysPair.HasChanges = true; + if (!beforeIsNull && (afterIsNull || !keysEqual)) + { + afterKeyPointer.IsUnlinkOld = true; + keysPair.HasChanges = true; + } + if (!afterIsNull && (beforeIsNull || !keysEqual)) + { + afterKeyPointer.IsLinkNew = true; + keysPair.HasChanges = true; + } } } @@ -188,20 +196,16 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor // if needed. On the Insert fast path, we don't want any allocations otherwise; changeTracker is null. var keyMemLen = this.keyPointerSize * this.PSFCount; var keyBytes = stackalloc byte[keyMemLen]; - - var flagsMemLen = this.PSFCount * sizeof(PSFResultFlags); - PSFResultFlags* flags = stackalloc PSFResultFlags[this.PSFCount]; var anyMatch = false; for (var ii = 0; ii < this.PSFCount; ++ii) { ref KeyPointer keyPointer = ref Unsafe.AsRef>(keyBytes + ii * this.keyPointerSize); - keyPointer.PrevAddress = Constants.kInvalidAddress; + keyPointer.PreviousAddress = Constants.kInvalidAddress; keyPointer.PsfOrdinal = (byte)ii; var key = this.psfDefinitions[ii].Execute(providerData); keyPointer.IsNull = !key.HasValue; - *(flags + ii) = key.HasValue ? PSFResultFlags.None : PSFResultFlags.IsNull; if (key.HasValue) { keyPointer.Key = key.Value; @@ -213,7 +217,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor return Status.OK; ref CompositeKey compositeKey = ref Unsafe.AsRef>(keyBytes); - var input = new PSFInputSecondary(0, this.Id, flags); + var input = new PSFInputSecondary(0, this.Id); var value = recordId; int groupOrdinal = -1; @@ -223,8 +227,8 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor if (phase == PSFExecutePhase.PreUpdate) { // Get a free group ref and store the "before" values. - ref GroupKeysPair groupKeysPair = ref changeTracker.FindGroupRef(this.Id); - StoreKeys(ref groupKeysPair.Before, keyBytes, keyMemLen, flags, flagsMemLen); + ref GroupCompositeKeyPair groupKeysPair = ref changeTracker.FindGroupRef(this.Id); + StoreKeys(ref groupKeysPair.Before, keyBytes, keyMemLen); return Status.OK; } @@ -237,8 +241,8 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor } else { - ref GroupKeysPair groupKeysPair = ref changeTracker.GetGroupRef(groupOrdinal); - StoreKeys(ref groupKeysPair.After, keyBytes, keyMemLen, flags, flagsMemLen); + ref GroupCompositeKeyPair groupKeysPair = ref changeTracker.GetGroupRef(groupOrdinal); + StoreKeys(ref groupKeysPair.After, keyBytes, keyMemLen); this.MarkChanges(ref groupKeysPair); // TODOtest: In debug, for initial dev, follow chains to assert the values match what is in the record's compositeKey if (!groupKeysPair.HasChanges) diff --git a/cs/src/core/Index/PSF/PSFInput.cs b/cs/src/core/Index/PSF/PSFInput.cs index 4c75703ba..f3be559f8 100644 --- a/cs/src/core/Index/PSF/PSFInput.cs +++ b/cs/src/core/Index/PSF/PSFInput.cs @@ -17,8 +17,6 @@ namespace FASTER.core /// constraint in PSFInputSecondary public interface IPSFInput { - unsafe void SetFlags(PSFResultFlags* resultFlags); - /// /// The ID of the for this operation. /// @@ -29,12 +27,6 @@ public interface IPSFInput /// int PsfOrdinal { get; set; } - /// - /// For Insert(), determine if the result of the at - /// was null; if so the value did not match and should not be stored in the key's chain. - /// - bool IsNullAt { get; } - /// /// For Delete() or Insert() done as part of RCU, this indicates if it the tombstone should be set for this record. /// @@ -70,11 +62,6 @@ public int PsfOrdinal set => throw new PSFInvalidOperationException("Not valid for Primary FasterFKV"); } - public void SetFlags(PSFResultFlags* resultFlags) - => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); - - public bool IsNullAt => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); - public bool IsDelete { get => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); @@ -93,14 +80,12 @@ public bool IsDelete public unsafe class PSFInputSecondary : IPSFInput, IDisposable where TPSFKey : struct { - internal PSFResultFlags* resultFlags; // TODO: Replace with KeyPointer.Flags; remember to clear update bits private SectorAlignedMemory keyPointerMem; - internal PSFInputSecondary(int psfOrdinal, long groupId, PSFResultFlags* flags = null) + internal PSFInputSecondary(int psfOrdinal, long groupId) { this.PsfOrdinal = psfOrdinal; this.GroupId = groupId; - this.resultFlags = flags; this.ReadLogicalAddress = Constants.kInvalidAddress; } @@ -109,25 +94,18 @@ internal void SetQueryKey(SectorAlignedBufferPool pool, KeyAccessor key // Create a varlen CompositeKey with just one item. This is ONLY used as the query key to QueryPSF. this.keyPointerMem = pool.Get(keyAccessor.KeyPointerSize); ref KeyPointer keyPointer = ref Unsafe.AsRef>(keyPointerMem.GetValidPointer()); - keyPointer.PrevAddress = Constants.kInvalidAddress; - keyPointer.PsfOrdinal = (ushort)this.PsfOrdinal; - keyPointer.Key = key; + keyPointer.Initialize(this.PsfOrdinal, ref key); } public long GroupId { get; } public int PsfOrdinal { get; set; } - public void SetFlags(PSFResultFlags* resultFlags) => this.resultFlags = resultFlags; - - public bool IsNullAt => this.resultFlags[this.PsfOrdinal].HasFlag(PSFResultFlags.IsNull); - public bool IsDelete { get; set; } public long ReadLogicalAddress { get; set; } - public ref TPSFKey QueryKeyRef - => ref Unsafe.AsRef(this.keyPointerMem.GetValidPointer()); + public ref TPSFKey QueryKeyRef => ref Unsafe.AsRef(this.keyPointerMem.GetValidPointer()); public void Dispose() { diff --git a/cs/src/core/Index/PSF/PSFOutput.cs b/cs/src/core/Index/PSF/PSFOutput.cs index ea305e755..a0e5cdf09 100644 --- a/cs/src/core/Index/PSF/PSFOutput.cs +++ b/cs/src/core/Index/PSF/PSFOutput.cs @@ -90,7 +90,7 @@ public PSFOperationStatus Visit(int psfOrdinal, ref TPSFKey key, ref CompositeKey compositeKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref key); ref KeyPointer keyPointer = ref this.keyAccessor.GetKeyPointerRef(ref compositeKey, psfOrdinal); Debug.Assert(keyPointer.PsfOrdinal == (ushort)psfOrdinal, "Visit found mismatched PSF ordinal"); - this.PreviousLogicalAddress = keyPointer.PrevAddress; + this.PreviousLogicalAddress = keyPointer.PreviousAddress; return new PSFOperationStatus(OperationStatus.SUCCESS); } @@ -103,7 +103,7 @@ public PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, this.Tombstone = tombstone; ref KeyPointer keyPointer = ref this.keyAccessor.GetKeyPointerRef(physicalAddress); Debug.Assert(keyPointer.PsfOrdinal == (ushort)psfOrdinal, "Visit found mismatched PSF ordinal"); - this.PreviousLogicalAddress = keyPointer.PrevAddress; + this.PreviousLogicalAddress = keyPointer.PreviousAddress; return new PSFOperationStatus(OperationStatus.SUCCESS); } } diff --git a/cs/src/core/Index/PSF/PSFResultFlags.cs b/cs/src/core/Index/PSF/PSFResultFlags.cs deleted file mode 100644 index 719e6dc73..000000000 --- a/cs/src/core/Index/PSF/PSFResultFlags.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -using System; - -namespace FASTER.core -{ - /// - /// Internal flags indicating how the keys returned from a are handled. - /// - [Flags] - public enum PSFResultFlags - { - /// - /// No flags set - /// - None = 0, - - /// - /// For Insert, this identifies a null PSF result (the record does not match the PSF and is not - /// included in any TPSFKey chain for it). Also used in - /// to determine whether to set . - /// - IsNull = 1, - - /// - /// For Update, the TPSFKey has changed; remove this record from the previous TPSFKey chain. - /// - UnlinkOld = 2, - - /// - /// For Update and insert, the TPSFKey has changed; add this record to the new TPSFKey chain. - /// - LinkNew = 4 - } -} From 2dcf7c6db84ad7d5104e146b29ccfa908d65712f Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Wed, 26 Aug 2020 11:58:59 -0700 Subject: [PATCH 16/19] First part of moving PSFs to functions-in-session --- cs/src/core/ClientSession/ClientSession.cs | 4 + cs/src/core/Index/FASTER/FASTERImpl.cs | 19 +- cs/src/core/Index/FASTER/FASTERThread.cs | 10 +- .../Index/PSF/FasterPSFContextOperations.cs | 68 +++---- cs/src/core/Index/PSF/FasterPSFImpl.cs | 100 +++++------ .../Index/PSF/FasterPSFSessionOperations.cs | 65 ++++--- cs/src/core/Index/PSF/PSFContext.cs | 14 ++ cs/src/core/Index/PSF/PSFFunctions.cs | 167 ++++++++++++++---- cs/src/core/Index/PSF/PSFGroup.cs | 107 ++++------- cs/src/core/Index/PSF/PSFInput.cs | 78 +++----- cs/src/core/Index/PSF/PSFOutput.cs | 39 ++-- cs/src/core/Index/PSF/PSFReadArgs.cs | 9 +- cs/src/core/Index/PSF/SessionManager.cs | 47 +++++ 13 files changed, 421 insertions(+), 306 deletions(-) create mode 100644 cs/src/core/Index/PSF/PSFContext.cs create mode 100644 cs/src/core/Index/PSF/SessionManager.cs diff --git a/cs/src/core/ClientSession/ClientSession.cs b/cs/src/core/ClientSession/ClientSession.cs index a2e84ef73..6621eae1a 100644 --- a/cs/src/core/ClientSession/ClientSession.cs +++ b/cs/src/core/ClientSession/ClientSession.cs @@ -66,6 +66,8 @@ internal ClientSession( Debug.WriteLine("Warning: Session param of variableLengthStruct provided for non-varlen allocator"); } + this.CreateLazyPsfSessionWrapper(); + // Session runs on a single thread if (!supportAsync) UnsafeResumeThread(); @@ -87,6 +89,8 @@ public void Dispose() // Session runs on a single thread if (!SupportAsync) UnsafeSuspendThread(); + + DisposeLazyPsfSessionWrapper(); } /// diff --git a/cs/src/core/Index/FASTER/FASTERImpl.cs b/cs/src/core/Index/FASTER/FASTERImpl.cs index debaa7c2a..7daa88337 100644 --- a/cs/src/core/Index/FASTER/FASTERImpl.cs +++ b/cs/src/core/Index/FASTER/FASTERImpl.cs @@ -1387,17 +1387,19 @@ internal OperationStatus InternalContinuePendingRead; + functions.VisitSecondaryRead(ref hlog.GetContextRecordKey(ref request), ref hlog.GetContextRecordValue(ref request), + ref pendingContext.input, ref pendingContext.output, tombstone: false, // checked above isConcurrent: false); } else if (pendingContext.type == OperationType.PSF_READ_ADDRESS) { - pendingContext.psfReadArgs.Output.Visit(pendingContext.psfReadArgs.Input.PsfOrdinal, - ref hlog.GetContextRecordKey(ref request), + var functions = fasterSession as IPSFFunctions; + functions.VisitSecondaryRead(ref hlog.GetContextRecordKey(ref request), ref hlog.GetContextRecordValue(ref request), + ref pendingContext.input, ref pendingContext.output, tombstone: false, // checked above isConcurrent: false); } @@ -1718,11 +1720,15 @@ internal Status HandleOperationStatus( break; case OperationType.PSF_READ_KEY: internalStatus = PsfInternalReadKey(ref pendingContext.key.Get(), - ref pendingContext.psfReadArgs, + ref pendingContext.input, + ref pendingContext.output, + ref pendingContext.userContext, ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum); break; case OperationType.PSF_READ_ADDRESS: - internalStatus = PsfInternalReadAddress(ref pendingContext.psfReadArgs, + internalStatus = PsfInternalReadAddress(ref pendingContext.input, + ref pendingContext.output, + ref pendingContext.userContext, ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum); break; case OperationType.UPSERT: @@ -1736,6 +1742,7 @@ ref pendingContext.value.Get(), internalStatus = PsfInternalInsert(ref pendingContext.key.Get(), ref pendingContext.value.Get(), ref pendingContext.input, + ref pendingContext.userContext, ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum); break; case OperationType.DELETE: diff --git a/cs/src/core/Index/FASTER/FASTERThread.cs b/cs/src/core/Index/FASTER/FASTERThread.cs index c699392ef..84abec279 100644 --- a/cs/src/core/Index/FASTER/FASTERThread.cs +++ b/cs/src/core/Index/FASTER/FASTERThread.cs @@ -236,7 +236,7 @@ internal void InternalCompleteRetryRequest)pendingContext.input; + var functions = fasterSession as IPSFFunctions; var updateOp = pendingContext.psfUpdateArgs.ChangeTracker.UpdateOp; - if (psfInput.IsDelete && (updateOp == UpdateOperation.IPU || updateOp == UpdateOperation.RCU)) + if (functions.IsDelete(ref pendingContext.input) && (updateOp == UpdateOperation.IPU || updateOp == UpdateOperation.RCU)) { // RCU Insert of a tombstoned old record is followed by Insert of the new record. - if (pendingContext.psfUpdateArgs.ChangeTracker.FindGroup(psfInput.GroupId, out var ordinal)) + if (pendingContext.psfUpdateArgs.ChangeTracker.FindGroup(functions.GroupId(ref pendingContext.input), out var ordinal)) { ref GroupCompositeKeyPair groupKeysPair = ref pendingContext.psfUpdateArgs.ChangeTracker.GetGroupRef(ordinal); GetAfterRecordId(pendingContext.psfUpdateArgs.ChangeTracker, ref value); var pcontext = default(PendingContext); - PsfRcuInsert(groupKeysPair.After, ref value, ref pendingContext.input, + PsfRcuInsert(groupKeysPair.After, ref value, ref pendingContext.input, ref pendingContext.userContext, ref pcontext, fasterSession, currentCtx, pendingContext.serialNum + 1); } } diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs index 17af1782d..109d683f3 100644 --- a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -12,12 +12,12 @@ public partial class FasterKV : FasterBase, IFasterKV where Value : new() { [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfReadKey(ref Key key, ref PSFReadArgs psfArgs, FasterSession fasterSession, - long serialNo, FasterExecutionContext sessionCtx) + internal Status ContextPsfReadKey(ref Key key, ref Input input, ref Output output, ref Context context, + FasterSession fasterSession, long serialNo, FasterExecutionContext sessionCtx) where FasterSession : IFasterSession { var pcontext = default(PendingContext); - var internalStatus = this.PsfInternalReadKey(ref key, ref psfArgs, ref pcontext, fasterSession, sessionCtx, serialNo); + var internalStatus = this.PsfInternalReadKey(ref key, ref input, ref output, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND ? (Status)internalStatus : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, fasterSession, internalStatus); @@ -29,20 +29,20 @@ internal Status ContextPsfReadKey(ref Key [MethodImpl(MethodImplOptions.AggressiveInlining)] internal ValueTask> ContextPsfReadKeyAsync( ClientSession clientSession, - ref Key key, ref PSFReadArgs psfArgs, long serialNo, FasterExecutionContext sessionCtx, - PSFQuerySettings querySettings) + ref Key key, ref Input input, ref Output output, ref Context context, long serialNo, + FasterExecutionContext sessionCtx, PSFQuerySettings querySettings) where Functions : IFunctions { - return ContextPsfReadAsync(clientSession, isKey: true, ref key, ref psfArgs, serialNo, sessionCtx, querySettings); + return ContextPsfReadAsync(clientSession, isKey: true, ref key, ref input, ref output, ref context, serialNo, sessionCtx, querySettings); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfReadAddress(ref PSFReadArgs psfArgs, FasterSession fasterSession, - long serialNo, FasterExecutionContext sessionCtx) + internal Status ContextPsfReadAddress(ref Input input, ref Output output, ref Context context, + FasterSession fasterSession, long serialNo, FasterExecutionContext sessionCtx) where FasterSession : IFasterSession { var pcontext = default(PendingContext); - var internalStatus = this.PsfInternalReadAddress(ref psfArgs, ref pcontext, fasterSession, sessionCtx, serialNo); + var internalStatus = this.PsfInternalReadAddress(ref input, ref output, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND ? (Status)internalStatus : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, fasterSession, internalStatus); @@ -54,22 +54,21 @@ internal Status ContextPsfReadAddress(ref [MethodImpl(MethodImplOptions.AggressiveInlining)] internal ValueTask> ContextPsfReadAddressAsync( ClientSession clientSession, - ref PSFReadArgs psfArgs, long serialNo, FasterExecutionContext sessionCtx, - PSFQuerySettings querySettings) + ref Input input, ref Output output, ref Context context, long serialNo, + FasterExecutionContext sessionCtx, PSFQuerySettings querySettings) where Functions : IFunctions { var key = default(Key); - return ContextPsfReadAsync(clientSession, isKey: false, ref key, ref psfArgs, serialNo, sessionCtx, querySettings); + return ContextPsfReadAsync(clientSession, isKey: false, ref key, ref input, ref output, ref context, serialNo, sessionCtx, querySettings); } internal ValueTask> ContextPsfReadAsync( ClientSession clientSession, bool isKey, - ref Key key, ref PSFReadArgs psfArgs, long serialNo, FasterExecutionContext sessionCtx, - PSFQuerySettings querySettings) + ref Key key, ref Input input, ref Output output, ref Context context, long serialNo, + FasterExecutionContext sessionCtx, PSFQuerySettings querySettings) where Functions : IFunctions { var pcontext = default(PendingContext); - var output = default(Output); var nextSerialNum = clientSession.ctx.serialNum + 1; if (clientSession.SupportAsync) clientSession.UnsafeResumeThread(); @@ -77,8 +76,8 @@ internal ValueTask> ContextPs { TryReadAgain: var internalStatus = isKey - ? this.PsfInternalReadKey(ref key, ref psfArgs, ref pcontext, clientSession.FasterSession, sessionCtx, serialNo) - : this.PsfInternalReadAddress(ref psfArgs, ref pcontext, clientSession.FasterSession, sessionCtx, serialNo); + ? this.PsfInternalReadKey(ref key, ref input, ref output, ref context, ref pcontext, clientSession.FasterSession, sessionCtx, serialNo) + : this.PsfInternalReadAddress(ref input, ref output, ref context, ref pcontext, clientSession.FasterSession, sessionCtx, serialNo); if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { return new ValueTask>(new ReadAsyncResult((Status)internalStatus, output)); @@ -107,13 +106,14 @@ internal ValueTask> ContextPs } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfInsert(ref Key key, ref Value value, ref Input input, + internal Status ContextPsfInsert(ref Key key, ref Value value, + ref Input input, ref Context context, FasterSession fasterSession, long serialNo, FasterExecutionContext sessionCtx) where FasterSession : IFasterSession { var pcontext = default(PendingContext); - var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, + var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND ? (Status)internalStatus @@ -124,19 +124,20 @@ internal Status ContextPsfInsert(ref Key } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextPsfUpdate(ref GroupCompositeKeyPair groupKeysPair, ref Value value, ref Input input, + internal Status ContextPsfUpdate(ref GroupCompositeKeyPair groupKeysPair, ref Value value, + ref Input input, ref Context context, FasterSession fasterSession, long serialNo, FasterExecutionContext sessionCtx, PSFChangeTracker changeTracker) where FasterSession : IFasterSession { var pcontext = default(PendingContext); - var psfInput = (IPSFInput)input; - var groupKeys = groupKeysPair.Before; - psfInput.IsDelete = true; - var internalStatus = this.PsfInternalInsert(ref groupKeys.CastToKeyRef(), ref value, ref input, + var functions = GetFunctions(ref context); + functions.SetDelete(ref input, true); + + var internalStatus = this.PsfInternalInsert(ref groupKeys.CastToKeyRef(), ref value, ref input, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); Status status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND ? (Status)internalStatus @@ -147,19 +148,19 @@ internal Status ContextPsfUpdate(GroupCompositeKey groupKeys, ref Value value, ref Input input, - ref PendingContext pcontext, FasterSession fasterSession, + ref Context context, ref PendingContext pcontext, FasterSession fasterSession, FasterExecutionContext sessionCtx, long serialNo) where FasterSession : IFasterSession { - var psfInput = (IPSFInput)input; - psfInput.IsDelete = false; - var internalStatus = this.PsfInternalInsert(ref groupKeys.CastToKeyRef(), ref value, ref input, + var functions = GetFunctions(ref context); + functions.SetDelete(ref input, false); + var internalStatus = this.PsfInternalInsert(ref groupKeys.CastToKeyRef(), ref value, ref input, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); return internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND ? (Status)internalStatus @@ -168,16 +169,17 @@ private Status PsfRcuInsert(GroupComposit [MethodImpl(MethodImplOptions.AggressiveInlining)] internal Status ContextPsfDelete(ref Key key, ref Value value, ref Input input, - FasterSession fasterSession, long serialNo, + ref Context context, FasterSession fasterSession, long serialNo, FasterExecutionContext sessionCtx, PSFChangeTracker changeTracker) where FasterSession : IFasterSession { var pcontext = default(PendingContext); - var psfInput = (IPSFInput)input; - psfInput.IsDelete = true; - var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, ref pcontext, fasterSession, sessionCtx, serialNo); + var functions = GetFunctions(ref context); + functions.SetDelete(ref input, true); + + var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); Status status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND ? (Status)internalStatus : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, fasterSession, internalStatus); diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index 856530c5e..3bbb883be 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -75,9 +75,12 @@ public unsafe PsfQueryKeyContainer(ref Key key, KeyAccessor keyAccessor, Se public void Dispose() => this.mem.Return(); } + private IPSFFunctions GetFunctions(ref Context context) + => (context as PSFContext).Functions as IPSFFunctions; + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal OperationStatus PsfInternalReadKey( - ref Key queryKeyPointerRefAsKeyRef, ref PSFReadArgs psfArgs, + ref Key queryKeyPointerRefAsKeyRef, ref Input input, ref Output output, ref Context context, ref PendingContext pendingContext, FasterSession fasterSession, FasterExecutionContext sessionCtx, long lsn) @@ -89,8 +92,7 @@ internal OperationStatus PsfInternalReadKey(ref context); ref KeyPointer queryKeyPointer = ref KeyPointer.CastFromKeyRef(ref queryKeyPointerRefAsKeyRef); var hash = this.PsfKeyAccessor.GetHashCode64(ref queryKeyPointer); @@ -120,9 +122,9 @@ internal OperationStatus PsfInternalReadKey= hlog.SafeReadOnlyAddress).Status; + return functions.VisitSecondaryRead(ref hlog.GetValue(recordAddress), ref input, ref output, physicalAddress, + hlog.GetInfo(recordAddress).Tombstone, + isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; } // On-Disk Region @@ -196,15 +197,14 @@ ref hlog.GetValue(recordAddress), { pendingContext.type = OperationType.PSF_READ_KEY; pendingContext.key = new PsfQueryKeyContainer(ref queryKeyPointerRefAsKeyRef, this.PsfKeyAccessor, this.hlog.bufferPool); - pendingContext.input = default; - pendingContext.output = default; + pendingContext.input = input; + pendingContext.output = output; pendingContext.userContext = default; pendingContext.entry.word = entry.word; pendingContext.logicalAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(hlog.GetPhysicalAddress(logicalAddress)); pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; pendingContext.heldLatch = heldOperation; - pendingContext.psfReadArgs = psfArgs; } #endregion @@ -213,7 +213,7 @@ ref hlog.GetValue(recordAddress), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal OperationStatus PsfInternalReadAddress( - ref PSFReadArgs psfArgs, + ref Input input, ref Output output, ref Context context, ref PendingContext pendingContext, FasterSession fasterSession, FasterExecutionContext sessionCtx, long lsn) @@ -226,18 +226,16 @@ internal OperationStatus PsfInternalReadAddress(ref context); -#region Look up record in in-memory HybridLog + #region Look up record in in-memory HybridLog // For PSFs, the addresses stored in the hash table point to KeyPointer entries, not the record header. - long logicalAddress = psfInput.ReadLogicalAddress; + long logicalAddress = functions.ReadLogicalAddress(ref input); PsfTrace($" ReadAddr: | {logicalAddress}"); #if false // TODOdcr: Support ReadCache in PSFs (must call this.PsfKeyAccessor.GetRecordAddressFromKeyLogicalAddress) + // TODO: PsfInternalReadAddress should handle ReadCache for primary FKV if (UseReadCache && ReadFromCache(ref logicalAddress, ref physicalAddress, ref latestRecordVersion)) { if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) @@ -253,7 +251,7 @@ ref readcache.GetValue(physicalAddress), if (logicalAddress >= hlog.HeadAddress) { - if (this.ImplmentsPSFs && !ScanQueryChain(ref logicalAddress, ref KeyPointer.CastFromKeyRef(ref psfInput.QueryKeyRef), ref latestRecordVersion)) + if (this.ImplmentsPSFs && !ScanQueryChain(ref logicalAddress, ref KeyPointer.CastFromKeyRef(ref functions.QueryKeyRef(ref input)), ref latestRecordVersion)) { goto ProcessAddress; // RECORD_ON_DISK or not found } @@ -269,19 +267,15 @@ ref readcache.GetValue(physicalAddress), // Mutable region (even fuzzy region is included here) is above SafeReadOnlyAddress and // is concurrent; Immutable region will not be changed. long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - if (!this.ImplmentsPSFs) + if (this.ImplmentsPSFs) { - return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), - ref hlog.GetValue(physicalAddress), - hlog.GetInfo(physicalAddress).Tombstone, - isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; + long recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + return functions.VisitSecondaryRead(ref hlog.GetValue(recordAddress), ref input, ref output, physicalAddress, + hlog.GetInfo(recordAddress).Tombstone, + isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; } - - long recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); - return psfOutput.Visit(psfInput.PsfOrdinal, physicalAddress, - ref hlog.GetValue(recordAddress), - hlog.GetInfo(recordAddress).Tombstone, - isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; + return functions.VisitPrimaryReadAddress(ref hlog.GetKey(physicalAddress), ref hlog.GetValue(physicalAddress), + ref output, isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; } // On-Disk Region @@ -293,7 +287,7 @@ ref hlog.GetValue(recordAddress), } else { - // No record found. TODOerr: we should always find the LogicalAddress + // No record found. TODOerr: we should not have called this function in this case. return OperationStatus.NOTFOUND; } @@ -304,10 +298,10 @@ ref hlog.GetValue(recordAddress), { pendingContext.type = OperationType.PSF_READ_ADDRESS; pendingContext.key = this.ImplmentsPSFs - ? new PsfQueryKeyContainer(ref psfInput.QueryKeyRef, this.PsfKeyAccessor, this.hlog.bufferPool) + ? new PsfQueryKeyContainer(ref functions.QueryKeyRef(ref input), this.PsfKeyAccessor, this.hlog.bufferPool) : default; - pendingContext.input = default; - pendingContext.output = default; + pendingContext.input = input; + pendingContext.output = output; pendingContext.userContext = default; pendingContext.entry.word = default; pendingContext.logicalAddress = this.ImplmentsPSFs @@ -316,7 +310,6 @@ ref hlog.GetValue(recordAddress), pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; pendingContext.heldLatch = LatchOperation.None; - pendingContext.psfReadArgs = psfArgs; } #endregion @@ -334,7 +327,7 @@ unsafe struct CASHelper [MethodImpl(MethodImplOptions.AggressiveInlining)] internal OperationStatus PsfInternalInsert( - ref Key firstKeyPointerRefAsKeyRef, ref Value value, ref Input input, + ref Key firstKeyPointerRefAsKeyRef, ref Value value, ref Input input, ref Context context, ref PendingContext pendingContext, FasterSession fasterSession, FasterExecutionContext sessionCtx, long lsn) @@ -343,7 +336,7 @@ internal OperationStatus PsfInternalInsert; + var functions = GetFunctions(ref context); ref CompositeKey compositeKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref firstKeyPointerRefAsKeyRef); // Update the KeyPointer links for chains with IsNullAt false (indicating a match with the @@ -354,24 +347,24 @@ internal OperationStatus PsfInternalInsert> Constants.kHashTagShift); if (sessionCtx.phase != Phase.REST) @@ -406,7 +399,7 @@ internal OperationStatus PsfInternalInsert storedKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref storedFirstKeyPointerRefAsKeyRef); @@ -447,13 +440,12 @@ internal OperationStatus PsfInternalInsert> Constants.kHashTagShift); - PsfTrace($" ({psfInput.PsfOrdinal}): {casHelper.hash} {tag} | newLA {newLogicalAddress} | prev {casHelper.entry.word}"); + PsfTrace($" ({psfOrdinal}): {casHelper.hash} {tag} | newLA {newLogicalAddress} | prev {casHelper.entry.word}"); if (casHelper.isNull) { PsfTraceLine(" null"); @@ -481,7 +473,7 @@ internal OperationStatus PsfInternalInsert { #region PSF calls for Secondary FasterKV - internal Status PsfInsert(ref Key key, ref Value value, ref Input input, long serialNo) + + // This value is created within the Primary FKV session. + Lazy, PSFOutputPrimaryReadAddress, PSFContext, PSFPrimaryFunctions>> psfLookupRecordIdSession; + + internal void CreateLazyPsfSessionWrapper() { + this.psfLookupRecordIdSession = new Lazy, PSFOutputPrimaryReadAddress, PSFContext, PSFPrimaryFunctions>>( + () => this.fht.NewSession, PSFOutputPrimaryReadAddress, PSFContext, PSFPrimaryFunctions>( + new PSFPrimaryFunctions())); + } + + internal void DisposeLazyPsfSessionWrapper() + { + if (!(this.psfLookupRecordIdSession is null) && this.psfLookupRecordIdSession.IsValueCreated) + this.psfLookupRecordIdSession.Value.Dispose(); + } + + private ClientSession, PSFOutputPrimaryReadAddress, PSFContext, PSFPrimaryFunctions> GetPsfLookupRecordSession() + => this.psfLookupRecordIdSession.Value; + + internal Status PsfInsert(ref Key key, ref Value value, ref Input input, ref Context context, long serialNo) { // Called on the secondary FasterKV if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextPsfInsert(ref key, ref value, ref input, this.FasterSession, serialNo, ctx); + return fht.ContextPsfInsert(ref key, ref value, ref input, ref context, this.FasterSession, serialNo, ctx); } finally { @@ -28,13 +47,13 @@ internal Status PsfInsert(ref Key key, ref Value value, ref Input input, long se } } - internal Status PsfReadKey(ref Key key, ref PSFReadArgs psfArgs, long serialNo) + internal Status PsfReadKey(ref Key key, ref Input input, ref Output output, ref Context context, long serialNo) { // Called on the secondary FasterKV if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextPsfReadKey(ref key, ref psfArgs, this.FasterSession, serialNo, ctx); + return fht.ContextPsfReadKey(ref key, ref input, ref output, ref context, this.FasterSession, serialNo, ctx); } finally { @@ -43,20 +62,19 @@ internal Status PsfReadKey(ref Key key, ref PSFReadArgs psfArgs, lon } internal ValueTask.ReadAsyncResult> PsfReadKeyAsync( - ref Key key, ref PSFReadArgs psfArgs, long serialNo, PSFQuerySettings querySettings) + ref Key key, ref Input input, ref Output output, ref Context context, long serialNo, PSFQuerySettings querySettings) { // Called on the secondary FasterKV - return fht.ContextPsfReadKeyAsync(this, ref key, ref psfArgs, serialNo, ctx, querySettings); + return fht.ContextPsfReadKeyAsync(this, ref key, ref input, ref output, ref context, serialNo, ctx, querySettings); } - - internal Status PsfReadAddress(ref PSFReadArgs psfArgs, long serialNo) + internal Status PsfReadAddress(ref Input input, ref Output output, ref Context context, long serialNo) { // Called on the secondary FasterKV if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextPsfReadAddress(ref psfArgs, this.FasterSession, serialNo, ctx); + return fht.ContextPsfReadAddress(ref input, ref output, ref context, this.FasterSession, serialNo, ctx); } finally { @@ -65,20 +83,21 @@ internal Status PsfReadAddress(ref PSFReadArgs psfArgs, long serialN } internal ValueTask.ReadAsyncResult> PsfReadAddressAsync( - ref PSFReadArgs psfArgs, long serialNo, PSFQuerySettings querySettings) + ref Input input, ref Output output, ref Context context, long serialNo, PSFQuerySettings querySettings) { // Called on the secondary FasterKV - return fht.ContextPsfReadAddressAsync(this, ref psfArgs, serialNo, ctx, querySettings); + return fht.ContextPsfReadAddressAsync(this, ref input, ref output, ref context, serialNo, ctx, querySettings); } - internal Status PsfUpdate(ref GroupCompositeKeyPair groupKeysPair, ref Value value, ref Input input, long serialNo, + internal Status PsfUpdate(ref GroupCompositeKeyPair groupKeysPair, ref Value value, ref Input input, + ref Context context, long serialNo, PSFChangeTracker changeTracker) { // Called on the secondary FasterKV if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextPsfUpdate(ref groupKeysPair, ref value, ref input, this.FasterSession, serialNo, ctx, changeTracker); + return fht.ContextPsfUpdate(ref groupKeysPair, ref value, ref input, ref context, this.FasterSession, serialNo, ctx, changeTracker); } finally { @@ -86,14 +105,14 @@ internal Status PsfUpdate(ref GroupCompositeKeyPair groupKeysPair } } - internal Status PsfDelete(ref Key key, ref Value value, ref Input input, long serialNo, + internal Status PsfDelete(ref Key key, ref Value value, ref Input input, ref Context context, long serialNo, PSFChangeTracker changeTracker) { // Called on the secondary FasterKV if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextPsfDelete(ref key, ref value, ref input, this.FasterSession, serialNo, ctx, changeTracker); + return fht.ContextPsfDelete(ref key, ref value, ref input, ref context, this.FasterSession, serialNo, ctx, changeTracker); } finally { @@ -108,9 +127,11 @@ internal Status PsfDelete(ref Key key, ref Value value, ref Input internal Status CreateProviderData(long logicalAddress, ConcurrentQueue> providerDatas) { // Looks up logicalAddress in the primary FasterKV - var primaryOutput = new PSFOutputPrimaryReadAddress(this.fht.hlog, providerDatas); - var psfArgs = new PSFReadArgs(new PSFInputPrimaryReadAddress(logicalAddress), primaryOutput); - return this.PsfReadAddress(ref psfArgs, this.ctx.serialNum + 1); + var output = new PSFOutputPrimaryReadAddress(this.fht.hlog, providerDatas); + var input = new PSFInputPrimaryReadAddress(logicalAddress); + var session = this.GetPsfLookupRecordSession(); + var context = new PSFContext { Functions = session.functions }; + return session.PsfReadAddress(ref input, ref output, ref context, this.ctx.serialNum + 1); } internal IEnumerable> ReturnProviderDatas(IEnumerable logicalAddresses) @@ -144,9 +165,11 @@ internal IEnumerable> ReturnProviderDatas(IEnum internal async ValueTask> CreateProviderDataAsync(long logicalAddress, ConcurrentQueue> providerDatas, PSFQuerySettings querySettings) { // Looks up logicalAddress in the primary FasterKV - var primaryOutput = new PSFOutputPrimaryReadAddress(this.fht.hlog, providerDatas); - var psfArgs = new PSFReadArgs(new PSFInputPrimaryReadAddress(logicalAddress), primaryOutput); - var readAsyncResult = await this.PsfReadAddressAsync(ref psfArgs, this.ctx.serialNum + 1, querySettings); + var output = new PSFOutputPrimaryReadAddress(this.fht.hlog, providerDatas); + var input = new PSFInputPrimaryReadAddress(logicalAddress); + var session = this.GetPsfLookupRecordSession(); + var context = new PSFContext { Functions = session.functions }; + var readAsyncResult = await session.PsfReadAddressAsync(ref input, ref output, ref context, this.ctx.serialNum + 1, querySettings); if (querySettings.IsCanceled) return null; var (status, _) = readAsyncResult.CompleteRead(); diff --git a/cs/src/core/Index/PSF/PSFContext.cs b/cs/src/core/Index/PSF/PSFContext.cs new file mode 100644 index 000000000..c2579bbe5 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFContext.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// Context for operations on the secondary FasterKV instance. + /// + public class PSFContext + { + // // TODO Hack because we can't get the functions object from the session, so pass this as context (also hacked to make it a class) + internal IPSFFunctions Functions; + } +} diff --git a/cs/src/core/Index/PSF/PSFFunctions.cs b/cs/src/core/Index/PSF/PSFFunctions.cs index d364f22a4..9194f2209 100644 --- a/cs/src/core/Index/PSF/PSFFunctions.cs +++ b/cs/src/core/Index/PSF/PSFFunctions.cs @@ -3,65 +3,170 @@ #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member -using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; namespace FASTER.core { + // Non-generic interface + internal interface IPSFFunctions { } + + internal interface IPSFFunctions : IPSFFunctions + { + #region Input accessors + + long GroupId(ref TInput input); + + bool IsDelete(ref TInput input); + bool SetDelete(ref TInput input, bool value); + + public long ReadLogicalAddress(ref TInput input); + + public ref TKey QueryKeyRef(ref TInput input); + #endregion Input accessors + + #region Output visitors + PSFOperationStatus VisitPrimaryReadAddress(ref TKey key, ref TValue value, ref TOutput output, bool isConcurrent); + + PSFOperationStatus VisitSecondaryRead(ref TValue value, ref TInput input, ref TOutput output, long physicalAddress, bool tombstone, bool isConcurrent); + + PSFOperationStatus VisitSecondaryRead(ref TKey key, ref TValue value, ref TInput input, ref TOutput output, bool tombstone, bool isConcurrent); + #endregion Output visitors + } + + /// + /// The Functions for the TRecordId (which is the Value param to the secondary FasterKV); mostly pass-through + /// + /// The type of the user key in the primary Faster KV + /// The type of the user value in the primary Faster KV + internal class PSFPrimaryFunctions : StubbedFunctions, PSFOutputPrimaryReadAddress>, + IPSFFunctions, PSFOutputPrimaryReadAddress> + where TKVKey : new() + where TKVValue : new() + { + public long GroupId(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + + public bool IsDelete(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + public bool SetDelete(ref PSFInputPrimaryReadAddress input, bool value) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long ReadLogicalAddress(ref PSFInputPrimaryReadAddress input) => input.ReadLogicalAddress; + + public ref TKVKey QueryKeyRef(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus VisitPrimaryReadAddress(ref TKVKey key, ref TKVValue value, ref PSFOutputPrimaryReadAddress output, bool isConcurrent) + { + // Tombstone is not needed here; it is only needed for the chains in the secondary FKV. + output.ProviderDatas.Enqueue(new FasterKVProviderData(output.allocator, ref key, ref value)); + return new PSFOperationStatus(OperationStatus.SUCCESS); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus VisitSecondaryRead(ref TKVValue value, ref PSFInputPrimaryReadAddress input, ref PSFOutputPrimaryReadAddress output, + long physicalAddress, bool tombstone, bool isConcurrent) + => throw new PSFInternalErrorException("Cannot call this form of Visit() on the primary FKV"); // TODO review error messages + + public PSFOperationStatus VisitSecondaryRead(ref TKVKey key, ref TKVValue value, ref PSFInputPrimaryReadAddress input, ref PSFOutputPrimaryReadAddress output, + bool tombstone, bool isConcurrent) + => throw new PSFInternalErrorException("Cannot call this form of Visit() on the primary FKV"); // TODO review error messages + } + /// /// The Functions for the TRecordId (which is the Value param to the secondary FasterKV); mostly pass-through /// /// The type of the result key /// The type of the value - public class PSFFunctions : IFunctions, - PSFOutputSecondary, Empty> - where TPSFKey: struct - where TRecordId: struct + public class PSFSecondaryFunctions : StubbedFunctions, PSFOutputSecondary>, + IPSFFunctions, PSFOutputSecondary> + where TPSFKey : new() + where TRecordId : new() { - // TODO: remove stuff that has been moved to PSFOutput.Visit, etc. + public long GroupId(ref PSFInputSecondary input) => input.GroupId; - #region Upserts - public bool ConcurrentWriter(ref TPSFKey _, ref TRecordId src, ref TRecordId dst) + public bool IsDelete(ref PSFInputSecondary input) => input.IsDelete; + public bool SetDelete(ref PSFInputSecondary input, bool value) => input.IsDelete = value; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long ReadLogicalAddress(ref PSFInputSecondary input) => input.ReadLogicalAddress; + + public ref TPSFKey QueryKeyRef(ref PSFInputSecondary input) => ref input.QueryKeyRef; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus VisitPrimaryReadAddress(ref TPSFKey key, ref TRecordId value, ref PSFOutputSecondary output, bool isConcurrent) + => throw new PSFInternalErrorException("Cannot call this form of Visit() on the secondary FKV"); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus VisitSecondaryRead(ref TRecordId value, ref PSFInputSecondary input, ref PSFOutputSecondary output, + long physicalAddress, bool tombstone, bool isConcurrent) { - dst = src; - return true; + // This is the secondary FKV; we hold onto the RecordId and create the provider data when QueryPSF returns. + output.RecordId = value; + output.Tombstone = tombstone; + ref KeyPointer keyPointer = ref output.keyAccessor.GetKeyPointerRef(physicalAddress); + Debug.Assert(keyPointer.PsfOrdinal == input.PsfOrdinal, "Visit found mismatched PSF ordinal"); + output.PreviousLogicalAddress = keyPointer.PreviousAddress; + return new PSFOperationStatus(OperationStatus.SUCCESS); } - public void SingleWriter(ref TPSFKey _, ref TRecordId src, ref TRecordId dst) - => dst = src; + public PSFOperationStatus VisitSecondaryRead(ref TPSFKey key, ref TRecordId value, ref PSFInputSecondary input, ref PSFOutputSecondary output, + bool tombstone, bool isConcurrent) + { + // This is the secondary FKV; we hold onto the RecordId and create the provider data when QueryPSF returns. + output.RecordId = value; + output.Tombstone = tombstone; + ref CompositeKey compositeKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref key); + ref KeyPointer keyPointer = ref output.keyAccessor.GetKeyPointerRef(ref compositeKey, input.PsfOrdinal); + Debug.Assert(keyPointer.PsfOrdinal == input.PsfOrdinal, "Visit found mismatched PSF ordinal"); + output.PreviousLogicalAddress = keyPointer.PreviousAddress; + return new PSFOperationStatus(OperationStatus.SUCCESS); + } + } + + public class StubbedFunctions : IFunctions + { + // All IFunctions methods are unused; the IFunctions "implementation" is to satisfy the ClientSession requirement. We use only the Visit methods. + #region IFunctions stubs + private const string MustUseVisitMethod = "PSF-implementing FasterKVs must use one of the Visit* methods"; + + #region Upserts + public bool ConcurrentWriter(ref TKey _, ref TValue src, ref TValue dst) => throw new PSFInternalErrorException(MustUseVisitMethod); + + public void SingleWriter(ref TKey _, ref TValue src, ref TValue dst) => throw new PSFInternalErrorException(MustUseVisitMethod); - public void UpsertCompletionCallback(ref TPSFKey _, ref TRecordId value, Empty ctx) - { /* TODO: UpsertCompletionCallback */ } + public void UpsertCompletionCallback(ref TKey _, ref TValue value, PSFContext ctx) => throw new PSFInternalErrorException(MustUseVisitMethod); #endregion Upserts #region Reads - public void ConcurrentReader(ref TPSFKey key, ref PSFInputSecondary input, ref TRecordId value, ref PSFOutputSecondary dst) - => throw new PSFInternalErrorException("PSFOutput.Visit instead of ConcurrentReader should be called on PSF-implementing FasterKVs"); + public void ConcurrentReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst) + => throw new PSFInternalErrorException(MustUseVisitMethod); - public unsafe void SingleReader(ref TPSFKey _, ref PSFInputSecondary input, ref TRecordId value, ref PSFOutputSecondary dst) - => throw new PSFInternalErrorException("PSFOutput.Visit instead of SingleReader should be called on PSF-implementing FasterKVs"); + public unsafe void SingleReader(ref TKey _, ref TInput input, ref TValue value, ref TOutput dst) + => throw new PSFInternalErrorException(MustUseVisitMethod); - public void ReadCompletionCallback(ref TPSFKey _, ref PSFInputSecondary input, ref PSFOutputSecondary output, Empty ctx, Status status) - { /* TODO: ReadCompletionCallback */ } + public void ReadCompletionCallback(ref TKey _, ref TInput input, ref TOutput output, PSFContext ctx, Status status) + => throw new PSFInternalErrorException(MustUseVisitMethod); #endregion Reads #region RMWs - public void CopyUpdater(ref TPSFKey _, ref PSFInputSecondary input, ref TRecordId oldValue, ref TRecordId newValue) - => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); + public void CopyUpdater(ref TKey _, ref TInput input, ref TValue oldValue, ref TValue newValue) + => throw new PSFInternalErrorException(MustUseVisitMethod); - public void InitialUpdater(ref TPSFKey _, ref PSFInputSecondary input, ref TRecordId value) - => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); + public void InitialUpdater(ref TKey _, ref TInput input, ref TValue value) + => throw new PSFInternalErrorException(MustUseVisitMethod); - public bool InPlaceUpdater(ref TPSFKey _, ref PSFInputSecondary input, ref TRecordId value) - => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); + public bool InPlaceUpdater(ref TKey _, ref TInput input, ref TValue value) + => throw new PSFInternalErrorException(MustUseVisitMethod); - public void RMWCompletionCallback(ref TPSFKey _, ref PSFInputSecondary input, Empty ctx, Status status) - => throw new PSFInternalErrorException("RMW should not be done on PSF-implementing FasterKVs"); + public void RMWCompletionCallback(ref TKey _, ref TInput input, PSFContext ctx, Status status) + => throw new PSFInternalErrorException(MustUseVisitMethod); #endregion RMWs - public void DeleteCompletionCallback(ref TPSFKey _, Empty ctx) - { /* TODO: DeleteCompletionCallback */ } + public void DeleteCompletionCallback(ref TKey _, PSFContext ctx) + => throw new PSFInternalErrorException(MustUseVisitMethod); public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) - { /* TODO: CheckpointCompletionCallback */ } + => throw new PSFInternalErrorException(MustUseVisitMethod); + #endregion IFunctions stubs } } diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs index 51fe14fc1..6ce68d38e 100644 --- a/cs/src/core/Index/PSF/PSFGroup.cs +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -3,7 +3,6 @@ using FASTER.core.Index.PSF; using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; @@ -26,7 +25,6 @@ public class PSFGroup : IExecutePSF fht; - private readonly PSFFunctions functions; internal IPSFDefinition[] psfDefinitions; private readonly PSFRegistrationSettings regSettings; @@ -39,18 +37,7 @@ public class PSFGroup : IExecutePSF)); private readonly int recordIdSize = (Utility.GetSize(default(TRecordId)) + sizeof(long) - 1) & ~(sizeof(long) - 1); - internal ConcurrentStack, - PSFOutputSecondary, Empty, PSFFunctions>> freeSessions - = new ConcurrentStack, - PSFOutputSecondary, Empty, PSFFunctions>>(); - internal ConcurrentBag, - PSFOutputSecondary, Empty, PSFFunctions>> allSessions - = new ConcurrentBag, - PSFOutputSecondary, Empty, PSFFunctions>>(); + private SessionManager, PSFOutputSecondary, PSFSecondaryFunctions> SecondarySessions; /// /// The list of s in this group @@ -93,7 +80,6 @@ public PSFGroup(PSFRegistrationSettings regSettings, IPSFDefinition(this.userKeyComparer, this.PSFCount, this.keyPointerSize); this.checkpointSettings = regSettings?.CheckpointSettings; - this.functions = new PSFFunctions(); this.fht = new FasterKV( regSettings.HashTableSize, regSettings.LogSettings, this.checkpointSettings, null /*SerializerSettings*/, new CompositeKey.UnusedKeyComparer(), @@ -103,30 +89,12 @@ public PSFGroup(PSFRegistrationSettings regSettings, IPSFDefinition, PSFOutputSecondary, PSFSecondaryFunctions>( + this.fht, regSettings.ThreadAffinitized); this.bufferPool = this.fht.hlog.bufferPool; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private ClientSession, - PSFOutputSecondary, Empty, - PSFFunctions> GetSession() - { - // Sessions are used only on post-RegisterPSF actions (Upsert, RMW, Query). - if (this.freeSessions.TryPop(out var session)) - return session; - session = this.fht.NewSession, PSFOutputSecondary, Empty, PSFFunctions>( - new PSFFunctions(), threadAffinitized: this.regSettings.ThreadAffinitized); - this.allSessions.Add(session); - return session; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ReleaseSession(ClientSession, - PSFOutputSecondary, Empty, - PSFFunctions> session) - => this.freeSessions.Push(session); - private IFasterEqualityComparer GetUserKeyComparer() { if (!(this.regSettings.KeyComparer is null)) @@ -217,7 +185,7 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor return Status.OK; ref CompositeKey compositeKey = ref Unsafe.AsRef>(keyBytes); - var input = new PSFInputSecondary(0, this.Id); + var input = new PSFInputSecondary(this.Id, 0); var value = recordId; int groupOrdinal = -1; @@ -253,23 +221,24 @@ public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recor // We don't need to do anything here for Delete. } - var session = this.GetSession(); + var session = this.SecondarySessions.GetSession(); try { var lsn = session.ctx.serialNum + 1; + var context = new PSFContext { Functions = session.functions }; return phase switch { - PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey.CastToFirstKeyPointerRefAsKeyRef(), ref value, ref input, lsn), + PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey.CastToFirstKeyPointerRefAsKeyRef(), ref value, ref input, ref context, lsn), PSFExecutePhase.PostUpdate => session.PsfUpdate(ref changeTracker.GetGroupRef(groupOrdinal), - ref value, ref input, lsn, changeTracker), - PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey.CastToFirstKeyPointerRefAsKeyRef(), ref value, ref input, lsn, + ref value, ref input, ref context, lsn, changeTracker), + PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey.CastToFirstKeyPointerRefAsKeyRef(), ref value, ref input, ref context, lsn, changeTracker), _ => throw new PSFInternalErrorException("Unknown PSF execution Phase {phase}") }; } finally { - this.ReleaseSession(session); + this.SecondarySessions.ReleaseSession(session); } } @@ -335,7 +304,7 @@ private unsafe PSFInputSecondary MakeQueryInput(int psfOrdinal, ref TPS { // Putting the query key in PSFInput is necessary because iterator functions cannot contain unsafe code or have // byref args, and bufferPool is needed here because the stack goes away as part of the iterator operation. - var psfInput = new PSFInputSecondary(psfOrdinal, this.Id); + var psfInput = new PSFInputSecondary(this.Id, psfOrdinal); psfInput.SetQueryKey(this.bufferPool, this.keyAccessor, ref key); return psfInput; } @@ -349,16 +318,15 @@ private IEnumerable Query(PSFInputSecondary input, PSFQueryS // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr. Also consider // having TPSFKeys[] for a single PSF walk through in parallel, so the FHT log memory access is sequential. - var secondaryOutput = new PSFOutputSecondary(this.keyAccessor); - var readArgs = new PSFReadArgs(input, secondaryOutput); - - var session = this.GetSession(); + var output = new PSFOutputSecondary(this.keyAccessor); + var session = this.SecondarySessions.GetSession(); + var context = new PSFContext { Functions = session.functions }; var deadRecs = new DeadRecords(); try { // Because we traverse the chain, we must wait for any pending read operations to complete. // TODOperf: See if there is a better solution than spinWaiting in CompletePending. - Status status = session.PsfReadKey(ref input.QueryKeyRef, ref readArgs, session.ctx.serialNum + 1); + Status status = session.PsfReadKey(ref input.QueryKeyRef, ref input, ref output, ref context, session.ctx.serialNum + 1); if (querySettings.IsCanceled) yield break; if (status == Status.PENDING) @@ -366,15 +334,15 @@ private IEnumerable Query(PSFInputSecondary input, PSFQueryS if (status != Status.OK) // TODOerr: check other status yield break; - if (secondaryOutput.Tombstone) - deadRecs.Add(secondaryOutput.RecordId); + if (output.Tombstone) + deadRecs.Add(output.RecordId); else - yield return secondaryOutput.RecordId; + yield return output.RecordId; do { - readArgs.Input.ReadLogicalAddress = secondaryOutput.PreviousLogicalAddress; - status = session.PsfReadAddress(ref readArgs, session.ctx.serialNum + 1); + input.ReadLogicalAddress = output.PreviousLogicalAddress; + status = session.PsfReadAddress(ref input, ref output, ref context, session.ctx.serialNum + 1); if (status == Status.PENDING) session.CompletePending(spinWait: true); if (querySettings.IsCanceled) @@ -382,15 +350,15 @@ private IEnumerable Query(PSFInputSecondary input, PSFQueryS if (status != Status.OK) // TODOerr: check other status yield break; - if (deadRecs.IsDead(secondaryOutput.RecordId, secondaryOutput.Tombstone)) + if (deadRecs.IsDead(output.RecordId, output.Tombstone)) continue; - yield return secondaryOutput.RecordId; - } while (secondaryOutput.PreviousLogicalAddress != Constants.kInvalidAddress); + yield return output.RecordId; + } while (output.PreviousLogicalAddress != Constants.kInvalidAddress); } finally { - this.ReleaseSession(session); + this.SecondarySessions.ReleaseSession(session); input.Dispose(); } } @@ -405,45 +373,44 @@ private async IAsyncEnumerable QueryAsync(PSFInputSecondary // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr. Also consider // having TPSFKeys[] for a single PSF walk through in parallel, so the FHT log memory access is sequential. - var secondaryOutput = new PSFOutputSecondary(this.keyAccessor); - var readArgs = new PSFReadArgs(input, secondaryOutput); - - var session = this.GetSession(); + var output = new PSFOutputSecondary(this.keyAccessor); + var session = this.SecondarySessions.GetSession(); + var context = new PSFContext { Functions = session.functions }; var deadRecs = new DeadRecords(); try { // Because we traverse the chain, we must wait for any pending read operations to complete. - var readAsyncResult = await session.PsfReadKeyAsync(ref input.QueryKeyRef, ref readArgs, session.ctx.serialNum + 1, querySettings); + var readAsyncResult = await session.PsfReadKeyAsync(ref input.QueryKeyRef, ref input, ref output, ref context, session.ctx.serialNum + 1, querySettings); if (querySettings.IsCanceled) yield break; var (status, _) = readAsyncResult.CompleteRead(); if (status != Status.OK) // TODOerr: check other status yield break; - if (secondaryOutput.Tombstone) - deadRecs.Add(secondaryOutput.RecordId); + if (output.Tombstone) + deadRecs.Add(output.RecordId); else - yield return secondaryOutput.RecordId; + yield return output.RecordId; do { - readArgs.Input.ReadLogicalAddress = secondaryOutput.PreviousLogicalAddress; - readAsyncResult = await session.PsfReadAddressAsync(ref readArgs, session.ctx.serialNum + 1, querySettings); + input.ReadLogicalAddress = output.PreviousLogicalAddress; + readAsyncResult = await session.PsfReadAddressAsync(ref input, ref output, ref context, session.ctx.serialNum + 1, querySettings); if (querySettings.IsCanceled) yield break; (status, _) = readAsyncResult.CompleteRead(); if (status != Status.OK) // TODOerr: check other status yield break; - if (deadRecs.IsDead(secondaryOutput.RecordId, secondaryOutput.Tombstone)) + if (deadRecs.IsDead(output.RecordId, output.Tombstone)) continue; - yield return secondaryOutput.RecordId; - } while (secondaryOutput.PreviousLogicalAddress != Constants.kInvalidAddress); + yield return output.RecordId; + } while (output.PreviousLogicalAddress != Constants.kInvalidAddress); } finally { - this.ReleaseSession(session); + this.SecondarySessions.ReleaseSession(session); input.Dispose(); } } diff --git a/cs/src/core/Index/PSF/PSFInput.cs b/cs/src/core/Index/PSF/PSFInput.cs index f3be559f8..9a8510445 100644 --- a/cs/src/core/Index/PSF/PSFInput.cs +++ b/cs/src/core/Index/PSF/PSFInput.cs @@ -8,84 +8,37 @@ namespace FASTER.core { - /// - /// The additional input interface passed to the PSF functions for internal Insert, Read, etc. operations. - /// - /// The type of the Key, either a for the - /// secondary FasterKV instances, or the user's TKVKey for the primary FasterKV instance. - /// The interface separation is needed for the PendingContext, and for the "TPSFKey : struct" - /// constraint in PSFInputSecondary - public interface IPSFInput - { - /// - /// The ID of the for this operation. - /// - long GroupId { get; } - - /// - /// The ordinal of the being queried; writable only for Insert. - /// - int PsfOrdinal { get; set; } - - /// - /// For Delete() or Insert() done as part of RCU, this indicates if it the tombstone should be set for this record. - /// - bool IsDelete { get; set; } - - /// - /// For tracing back the chain, this is the next logicalAddress to get. - /// - long ReadLogicalAddress { get; set; } - - ref TKey QueryKeyRef { get; } - } - /// /// Input to PsfInternalReadAddress on the primary (stores user values) FasterKV to retrieve the Key and Value /// for a logicalAddress returned from the secondary FasterKV instances. This class is FasterKV-provider-specific. /// /// The type of the key for user values - public unsafe class PSFInputPrimaryReadAddress : IPSFInput + public unsafe struct PSFInputPrimaryReadAddress + where TKey : new() { internal PSFInputPrimaryReadAddress(long readLA) { this.ReadLogicalAddress = readLA; } - /// - public long GroupId => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); - - /// - public int PsfOrdinal - { - get => Constants.kInvalidPsfOrdinal; - set => throw new PSFInvalidOperationException("Not valid for Primary FasterFKV"); - } - - public bool IsDelete - { - get => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); - set => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); - } - public long ReadLogicalAddress { get; set; } - - public ref TKey QueryKeyRef => throw new PSFInvalidOperationException("Not valid for Primary FasterKV"); } /// /// Input to operations on the secondary FasterKV instance (stores PSF chains) for everything /// except reading based on a LogicalAddress. /// - public unsafe class PSFInputSecondary : IPSFInput, IDisposable - where TPSFKey : struct + public unsafe struct PSFInputSecondary : IDisposable + where TPSFKey : new() { private SectorAlignedMemory keyPointerMem; - internal PSFInputSecondary(int psfOrdinal, long groupId) + internal PSFInputSecondary(long groupId, int psfOrdinal) { - this.PsfOrdinal = psfOrdinal; + this.keyPointerMem = null; this.GroupId = groupId; + this.PsfOrdinal = psfOrdinal; + this.IsDelete = false; this.ReadLogicalAddress = Constants.kInvalidAddress; } @@ -97,14 +50,29 @@ internal void SetQueryKey(SectorAlignedBufferPool pool, KeyAccessor key keyPointer.Initialize(this.PsfOrdinal, ref key); } + /// + /// The ID of the for this operation. + /// public long GroupId { get; } + /// + /// The ordinal of the in the group for this operation. + /// public int PsfOrdinal { get; set; } + /// + /// Whether this is a Delete (or the Delete part of an RCU) + /// public bool IsDelete { get; set; } + /// + /// The logical address to read in one of the PsfRead*Address methods + /// public long ReadLogicalAddress { get; set; } + /// + /// The query key for a QueryPSF method + /// public ref TPSFKey QueryKeyRef => ref Unsafe.AsRef(this.keyPointerMem.GetValidPointer()); public void Dispose() diff --git a/cs/src/core/Index/PSF/PSFOutput.cs b/cs/src/core/Index/PSF/PSFOutput.cs index a0e5cdf09..1724ca73e 100644 --- a/cs/src/core/Index/PSF/PSFOutput.cs +++ b/cs/src/core/Index/PSF/PSFOutput.cs @@ -4,36 +4,21 @@ #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member using System.Collections.Concurrent; -using System.Diagnostics; using System.Runtime.CompilerServices; namespace FASTER.core { - /// - /// The additional output interface passed to the PSF functions for internal Insert, Read, etc. operations. - /// - /// The type of the Key, either a for the - /// secondary FasterKV instances, or the user's TKVKey for the primary FasterKV instance. - /// The type of the Value, either a TRecordId for the - /// secondary FasterKV instances, or the user's TKVValue for the primary FasterKV instance. - public interface IPSFOutput - { - PSFOperationStatus Visit(int psfOrdinal, ref TKey key, ref TValue value, bool tombstone, bool isConcurrent); - - PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, ref TValue value, bool tombstone, bool isConcurrent); - } - /// /// Output from ReadInternal on the primary (stores user values) FasterKV instance when reading /// based on a LogicalAddress rather than a key. This class is FasterKV-provider-specific. /// /// The type of the key for user values /// The type of the user values - public unsafe class PSFOutputPrimaryReadAddress : IPSFOutput + public unsafe struct PSFOutputPrimaryReadAddress where TKVKey : new() where TKVValue : new() { - private readonly AllocatorBase allocator; + internal readonly AllocatorBase allocator; internal ConcurrentQueue> ProviderDatas { get; private set; } @@ -43,7 +28,7 @@ internal PSFOutputPrimaryReadAddress(AllocatorBase alloc, this.allocator = alloc; this.ProviderDatas = provDatas; } - +#if false [MethodImpl(MethodImplOptions.AggressiveInlining)] public PSFOperationStatus Visit(int psfOrdinal, ref TKVKey key, ref TKVValue value, bool tombstone, bool isConcurrent) { @@ -54,6 +39,7 @@ public PSFOperationStatus Visit(int psfOrdinal, ref TKVKey key, ref TKVValue val public PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, ref TKVValue value, bool tombstone, bool isConcurrent) => throw new PSFInternalErrorException("Cannot call this form of Visit() on the primary FKV"); // TODO review error messages +#endif } /// @@ -61,26 +47,28 @@ public PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, ref TKVVal /// /// The type of the key returned from a /// The type of the provider's record identifier - public unsafe class PSFOutputSecondary : IPSFOutput - where TPSFKey : struct - where TRecordId : struct + public unsafe struct PSFOutputSecondary + where TPSFKey : new() + where TRecordId : new() { - private readonly KeyAccessor keyAccessor; + internal readonly KeyAccessor keyAccessor; - internal TRecordId RecordId { get; private set; } + internal TRecordId RecordId { get; set; } - internal bool Tombstone { get; private set; } + internal bool Tombstone { get; set; } - internal long PreviousLogicalAddress { get; private set; } + internal long PreviousLogicalAddress { get; set; } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal PSFOutputSecondary(KeyAccessor keyAcc) { this.keyAccessor = keyAcc; this.RecordId = default; + this.Tombstone = false; this.PreviousLogicalAddress = Constants.kInvalidAddress; } +#if false public PSFOperationStatus Visit(int psfOrdinal, ref TPSFKey key, ref TRecordId value, bool tombstone, bool isConcurrent) { @@ -106,5 +94,6 @@ public PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, this.PreviousLogicalAddress = keyPointer.PreviousAddress; return new PSFOperationStatus(OperationStatus.SUCCESS); } +#endif } } diff --git a/cs/src/core/Index/PSF/PSFReadArgs.cs b/cs/src/core/Index/PSF/PSFReadArgs.cs index 2635183f3..755d5f607 100644 --- a/cs/src/core/Index/PSF/PSFReadArgs.cs +++ b/cs/src/core/Index/PSF/PSFReadArgs.cs @@ -3,15 +3,14 @@ namespace FASTER.core { + // TODO: Move out of PSFs and include the new chained-LA Read function. internal struct PSFReadArgs { - internal readonly IPSFInput Input; - internal readonly IPSFOutput Output; + internal readonly long LivenessCheckLogicalAddress; - internal PSFReadArgs(IPSFInput input, IPSFOutput output) + internal PSFReadArgs(long livenessCheckAddress) { - this.Input = input; - this.Output = output; + this.LivenessCheckLogicalAddress = livenessCheckAddress; } } } diff --git a/cs/src/core/Index/PSF/SessionManager.cs b/cs/src/core/Index/PSF/SessionManager.cs new file mode 100644 index 000000000..696ca50a5 --- /dev/null +++ b/cs/src/core/Index/PSF/SessionManager.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + // TODO: How are the sessions disposed? + class SessionManager + where TKey : struct + where TValue : struct + where TFunctions : IFunctions, new() + { + private readonly ConcurrentStack> freeSessions + = new ConcurrentStack>(); + private readonly ConcurrentBag> allSessions + = new ConcurrentBag>(); + + internal FasterKV fht; + private readonly bool threadAffinitized; + + internal SessionManager(FasterKV fht, bool threadAff) + { + this.fht = fht; + this.threadAffinitized = threadAff; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ClientSession GetSession() + { + // Sessions are used only on post-RegisterPSF actions (Upsert, RMW, Query). + if (this.freeSessions.TryPop(out var session)) + return session; + session = this.fht.NewSession(new TFunctions(), threadAffinitized: this.threadAffinitized); + this.allSessions.Add(session); + return session; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ReleaseSession(ClientSession session) + { + // TODO: Cap on number of saved sessions? + this.freeSessions.Push(session); + } + } +} From f265b3cfa1c1c220c9eac55855b508e8e93d6122 Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Thu, 3 Sep 2020 20:36:13 -0700 Subject: [PATCH 17/19] Make PSF methods public; first cut at README --- cs/src/core/Index/PSF/KeyPointer.cs | 2 +- cs/src/core/Index/PSF/PSFManager.cs | 344 ++++++++++++++++-- .../core/Index/PSF/PSFRegistrationSettings.cs | 6 +- cs/src/core/Index/PSF/README.md | 136 +++++++ 4 files changed, 459 insertions(+), 29 deletions(-) create mode 100644 cs/src/core/Index/PSF/README.md diff --git a/cs/src/core/Index/PSF/KeyPointer.cs b/cs/src/core/Index/PSF/KeyPointer.cs index 0c6c575fe..137945648 100644 --- a/cs/src/core/Index/PSF/KeyPointer.cs +++ b/cs/src/core/Index/PSF/KeyPointer.cs @@ -94,10 +94,10 @@ internal int OffsetToStartOfKeys get => this.offsetToStartOfKeys; set => this.offsetToStartOfKeys = (ushort)value; } - #endregion Accessors [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void ClearUpdateFlags() => this.flags = (byte)(this.flags & ~(kUnlinkOldBit | kLinkNewBit)); + #endregion Accessors [MethodImpl(MethodImplOptions.AggressiveInlining)] internal unsafe static ref KeyPointer CastFromKeyRef(ref TPSFKey keyRef) diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs index b7a0ebea6..635816923 100644 --- a/cs/src/core/Index/PSF/PSFManager.cs +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -14,7 +14,12 @@ namespace FASTER.core { - internal class PSFManager where TRecordId : struct, IComparable + /// + /// The class that manages PSFs. Called internally by the primary FasterKV. + /// + /// The type of the provider data returned by PSF queries; for the primary FasterKV, it is + /// The type of the Record identifier in the data provider; for the primary FasterKV it is the record's logical address + public class PSFManager where TRecordId : struct, IComparable { private readonly ConcurrentDictionary> psfGroups = new ConcurrentDictionary>(); @@ -23,7 +28,14 @@ private readonly ConcurrentDictionary this.psfGroups.Count > 0; - internal Status Upsert(TProviderData data, TRecordId recordId, PSFChangeTracker changeTracker) + /// + /// Inserts a new PSF key/RecordId, or adds the RecordId to an existing chain + /// + /// The provider's data; will be passed to the PSF execution + /// The record Id to be stored for any matching PSFs + /// Tracks changes if this is an existing Key/RecordId entry + /// A status code indicating the result of the operation + public Status Upsert(TProviderData data, TRecordId recordId, PSFChangeTracker changeTracker) { // TODO: RecordId locking, to ensure consistency of multiple PSFs if the same record is updated // multiple times; possibly a single Array[N] which is locked on TRecordId.GetHashCode % N. @@ -47,7 +59,12 @@ internal Status Upsert(TProviderData data, TRecordId recordId, PSFChangeTracker< return this.Update(changeTracker); } - internal Status Update(PSFChangeTracker changeTracker) + /// + /// Updates a PSF key/RecordId entry, possibly by RCU (Read-Copy-Update) + /// + /// Tracks changes for an existing Key/RecordId entry + /// A status code indicating the result of the operation + public Status Update(PSFChangeTracker changeTracker) { foreach (var group in this.psfGroups.Values) { @@ -60,7 +77,12 @@ internal Status Update(PSFChangeTracker changeTracker) return Status.OK; } - internal Status Delete(PSFChangeTracker changeTracker) + /// + /// Deletes a PSF key/RecordId entry from the chain, possibly by insertion of a "marked deleted" record + /// + /// Tracks changes for an existing Key/RecordId entry + /// A status code indicating the result of the operation + public Status Delete(PSFChangeTracker changeTracker) { foreach (var group in this.psfGroups.Values) { @@ -73,11 +95,28 @@ internal Status Delete(PSFChangeTracker changeTracker) return Status.OK; } - internal string[][] GetRegisteredPSFNames() => throw new NotImplementedException("TODO"); + /// + /// Obtains a list of registered PSF names organized by the groups defined in previous RegisterPSF calls. + /// + /// A list of registered PSF names organized by the groups defined in previous RegisterPSF calls. + public string[][] GetRegisteredPSFNames() => throw new NotImplementedException("TODO"); - internal PSFChangeTracker CreateChangeTracker() + /// + /// Creates an instance of a to track changes for an existing Key/RecordId entry. + /// + /// An instance of a to track changes for an existing Key/RecordId entry. + public PSFChangeTracker CreateChangeTracker() => new PSFChangeTracker(this.psfGroups.Values.Select(group => group.Id)); + /// + /// Sets the data for the state of a provider's data record prior to an update. + /// + /// Tracks changes for the Key/RecordId entry that will be updated. + /// The provider's data prior to the update; will be passed to the PSF execution + /// The record Id to be stored for any matching PSFs + /// Whether PSFs should be executed now or deferred. Should be 'true' if the provider's value type is an Object, + /// because the update will likely change the object's internal values, and thus a deferred 'before' execution will pick up the updated values instead. + /// A status code indicating the result of the operation public Status SetBeforeData(PSFChangeTracker changeTracker, TProviderData data, TRecordId recordId, bool executePSFsNow) { changeTracker.SetBeforeData(data, recordId); @@ -96,6 +135,13 @@ public Status SetBeforeData(PSFChangeTracker changeTra return Status.OK; } + /// + /// Sets the data for the state of a provider's data record after to an update. + /// + /// Tracks changes for the Key/RecordId entry that will be updated. + /// The provider's data after to the update; will be passed to the PSF execution + /// The record Id to be stored for any matching PSFs + /// A status code indicating the result of the operation public Status SetAfterData(PSFChangeTracker changeTracker, TProviderData data, TRecordId recordId) { changeTracker.SetAfterData(data, recordId); @@ -176,7 +222,14 @@ private static void VerifyRegistrationSettings(PSFRegistrationSettings< throw new PSFArgumentException("PSFs do not support ReadCache or CopyReadsToTail"); } - internal IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, IPSFDefinition def) + /// + /// Register a with a simple definition. + /// + /// The type of the key returned from the + /// Registration settings for the secondary FasterKV instances, etc. + /// The PSF definition + /// A PSF implementation( + public IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, IPSFDefinition def) where TPSFKey : struct { this.VerifyIsBlittable(); @@ -198,7 +251,14 @@ internal IPSF RegisterPSF(PSFRegistrationSettings registration } } - internal IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, IPSFDefinition[] defs) + /// + /// Register multiple s with a vector of definitions. + /// + /// The type of the key returned from the + /// Registration settings for the secondary FasterKV instances, etc. + /// The PSF definitions + /// A PSF implementation( + public IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, IPSFDefinition[] defs) where TPSFKey : struct { this.VerifyIsBlittable(); @@ -237,7 +297,15 @@ internal IPSF[] RegisterPSF(PSFRegistrationSettings registrati } } - internal IEnumerable QueryPSF(IPSF psf, TPSFKey key, PSFQuerySettings querySettings) + /// + /// Does a synchronous scan of a single PSF for records matching a single key + /// + /// The type of the key returned from the + /// The PSF to be queried + /// The identifying the records to be retrieved + /// Options for the PSF query operation + /// An enumeration of the s matching + public IEnumerable QueryPSF(IPSF psf, TPSFKey key, PSFQuerySettings querySettings) where TPSFKey : struct { var psfImpl = this.GetImplementingPSF(psf); @@ -251,7 +319,15 @@ internal IEnumerable QueryPSF(IPSF psf, TPSFKey key, PSFQuer } #if DOTNETCORE - internal async IAsyncEnumerable QueryPSFAsync(IPSF psf, TPSFKey key, PSFQuerySettings querySettings) + /// + /// Does an asynchronous scan of a single PSF for records matching a single key + /// + /// The type of the key returned from the + /// The PSF to be queried + /// The identifying the records to be retrieved + /// Options for the PSF query operation + /// An async enumeration of the s matching + public async IAsyncEnumerable QueryPSFAsync(IPSF psf, TPSFKey key, PSFQuerySettings querySettings) where TPSFKey : struct { var psfImpl = this.GetImplementingPSF(psf); @@ -266,7 +342,15 @@ internal async IAsyncEnumerable QueryPSFAsync(IPSF psf, TPSF #endif // DOTNETCORE - internal IEnumerable QueryPSF(IPSF psf, IEnumerable keys, PSFQuerySettings querySettings) + /// + /// Does a synchronous scan of a single PSF for records matching any of multiple keys, unioning the results. + /// + /// The type of the key returned from the + /// The PSF to be queried + /// The s identifying the records to be retrieved + /// Options for the PSF query operation + /// An enumeration of the s matching + public IEnumerable QueryPSF(IPSF psf, IEnumerable keys, PSFQuerySettings querySettings) where TPSFKey : struct { this.VerifyIsOurPSF(psf); @@ -286,7 +370,15 @@ internal IEnumerable QueryPSF(IPSF psf, IEnumerable } #if DOTNETCORE - internal async IAsyncEnumerable QueryPSFAsync(IPSF psf, IEnumerable keys, PSFQuerySettings querySettings) + /// + /// Does an asynchronous scan of a single PSF for records matching any of multiple keys, unioning the results. + /// + /// The type of the key returned from the + /// The PSF to be queried + /// The s identifying the records to be retrieved + /// Options for the PSF query operation + /// An async enumeration of the s matching + public async IAsyncEnumerable QueryPSFAsync(IPSF psf, IEnumerable keys, PSFQuerySettings querySettings) where TPSFKey : struct { this.VerifyIsOurPSF(psf); @@ -307,7 +399,20 @@ internal async IAsyncEnumerable QueryPSFAsync(IPSF psf, IEnu #endif // DOTNETCORE - internal IEnumerable QueryPSF( + /// + /// Does a synchronous scan of one key on each of two PSFs, returning records matching these keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The first PSF to be queried + /// The second PSF to be queried + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( IPSF psf1, TPSFKey1 key1, IPSF psf2, TPSFKey2 key2, Func matchPredicate, @@ -323,7 +428,20 @@ internal IEnumerable QueryPSF( } #if DOTNETCORE - internal IAsyncEnumerable QueryPSFAsync( + /// + /// Does an synchronous scan of one key on each of two PSFs, returning records matching these keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The first PSF to be queried + /// The second PSF to be queried + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( IPSF psf1, TPSFKey1 key1, IPSF psf2, TPSFKey2 key2, Func matchPredicate, @@ -340,7 +458,20 @@ internal IAsyncEnumerable QueryPSFAsync( #endif // DOTNETCORE - internal IEnumerable QueryPSF( + /// + /// Does a synchronous scan of multiple keys on each of two PSFs, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The first PSF to be queried + /// The second PSF to be queried + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( IPSF psf1, IEnumerable keys1, IPSF psf2, IEnumerable keys2, Func matchPredicate, @@ -356,7 +487,20 @@ internal IEnumerable QueryPSF( } #if DOTNETCORE - internal IAsyncEnumerable QueryPSFAsync( + /// + /// Does an asynchronous scan of multiple keys on each of two PSFs, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The first PSF to be queried + /// The second PSF to be queried + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( IPSF psf1, IEnumerable keys1, IPSF psf2, IEnumerable keys2, Func matchPredicate, @@ -372,6 +516,22 @@ internal IAsyncEnumerable QueryPSFAsync( } #endif // DOTNETCORE + /// + /// Does a synchronous scan of one key on each of three PSFs, returning records matching these keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The type of the key returned from the third + /// The first PSF to be queried + /// The second PSF to be queried + /// The third PSF to be queried + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and public IEnumerable QueryPSF( IPSF psf1, TPSFKey1 key1, IPSF psf2, TPSFKey2 key2, @@ -391,6 +551,22 @@ public IEnumerable QueryPSF( } #if DOTNETCORE + /// + /// Does an asynchronous scan of one key on each of three PSFs, returning records matching these keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The type of the key returned from the third + /// The first PSF to be queried + /// The second PSF to be queried + /// The third PSF to be queried + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and public IAsyncEnumerable QueryPSFAsync( IPSF psf1, TPSFKey1 key1, IPSF psf2, TPSFKey2 key2, @@ -410,6 +586,22 @@ public IAsyncEnumerable QueryPSFAsync( } #endif // DOTNETCORE + /// + /// Does a synchronous scan of multiple keys on each of three PSFs, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The type of the key returned from the third + /// The first PSF to be queried + /// The second PSF to be queried + /// The third PSF to be queried + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and public IEnumerable QueryPSF( IPSF psf1, IEnumerable keys1, IPSF psf2, IEnumerable keys2, @@ -429,6 +621,22 @@ public IEnumerable QueryPSF( } #if DOTNETCORE + /// + /// Does an asynchronous scan of multiple keys on each of three PSFs, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The type of the key returned from the third + /// The first PSF to be queried + /// The second PSF to be queried + /// The third PSF to be queried + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and public IAsyncEnumerable QueryPSFAsync( IPSF psf1, IEnumerable keys1, IPSF psf2, IEnumerable keys2, @@ -448,9 +656,18 @@ public IAsyncEnumerable QueryPSFAsync( } #endif // DOTNETCORE - // Power user versions. Anything more complicated than this can be post-processed with LINQ. + // Power user versions. Anything more complicated than these can be post-processed with LINQ. - internal IEnumerable QueryPSF( + /// + /// Does a synchronous scan of multiple keys on each of multiple PSFs with the same type, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the s + /// An enumeration of tuples containing a and the s to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, Func matchPredicate, PSFQuerySettings querySettings = null) @@ -464,7 +681,16 @@ internal IEnumerable QueryPSF( } #if DOTNETCORE - internal IAsyncEnumerable QueryPSFAsync( + /// + /// Does an asynchronous scan of multiple keys on each of multiple PSFs with the same type, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the s + /// An enumeration of tuples containing a and the s to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, Func matchPredicate, PSFQuerySettings querySettings = null) @@ -478,7 +704,18 @@ internal IAsyncEnumerable QueryPSFAsync( } #endif // DOTNETCORE - internal IEnumerable QueryPSF( + /// + /// Does a synchronous scan of multiple keys on each of multiple PSFs on each of two TPSFKey types, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first set of s + /// The type of the key returned from the second set of s + /// The first enumeration of tuples containing a and the TPSFKey to be queried on it + /// The second enumeration of tuples containing a and the TPSFKey to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, Func matchPredicate, @@ -495,7 +732,18 @@ internal IEnumerable QueryPSF( } #if DOTNETCORE - internal IAsyncEnumerable QueryPSFAsync( + /// + /// Does an asynchronous scan of multiple keys on each of multiple PSFs on each of two TPSFKey types, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first set of s + /// The type of the key returned from the second set of s + /// The first enumeration of tuples containing a and the TPSFKey to be queried on it + /// The second enumeration of tuples containing a and the TPSFKey to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, Func matchPredicate, @@ -512,7 +760,20 @@ internal IAsyncEnumerable QueryPSFAsync( } #endif // DOTNETCORE - internal IEnumerable QueryPSF( + /// + /// Does a synchronous scan of multiple keys on each of multiple PSFs on each of three TPSFKey types, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first set of s + /// The type of the key returned from the second set of s + /// The type of the key returned from the third set of s + /// The first enumeration of tuples containing a and the TPSFKey to be queried on it + /// The second enumeration of tuples containing a and the TPSFKey to be queried on it + /// The third enumeration of tuples containing a and the TPSFKey to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys3, @@ -532,7 +793,20 @@ internal IEnumerable QueryPSF( } #if DOTNETCORE - internal IAsyncEnumerable QueryPSFAsync( + /// + /// Does an asynchronous scan of multiple keys on each of multiple PSFs on each of three TPSFKey types, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first set of s + /// The type of the key returned from the second set of s + /// The type of the key returned from the third set of s + /// The first enumeration of tuples containing a and the TPSFKey to be queried on it + /// The second enumeration of tuples containing a and the TPSFKey to be queried on it + /// The third enumeration of tuples containing a and the TPSFKey to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys3, @@ -554,21 +828,37 @@ internal IAsyncEnumerable QueryPSFAsync #region Checkpoint Operations // TODO Separate Tasks for each group's commit/restore operations? + + /// + /// For each , take a full checkpoint of the FasterKV implementing the group's PSFs. + /// public bool TakeFullCheckpoint() => this.psfGroups.Values.Aggregate(true, (result, group) => group.TakeFullCheckpoint() && result); + /// + /// For each , complete ongoing checkpoint (spin-wait) + /// public Task CompleteCheckpointAsync(CancellationToken token = default) { var tasks = this.psfGroups.Values.Select(group => group.CompleteCheckpointAsync(token).AsTask()).ToArray(); return Task.WhenAll(tasks); } + /// + /// For each , take a checkpoint of the Index (hashtable) only + /// public bool TakeIndexCheckpoint() => this.psfGroups.Values.Aggregate(true, (result, group) => group.TakeIndexCheckpoint() && result); + /// + /// For each , take a checkpoint of the hybrid log only + /// public bool TakeHybridLogCheckpoint() => this.psfGroups.Values.Aggregate(true, (result, group) => group.TakeHybridLogCheckpoint() && result); + /// + /// For each , recover from last successful checkpoints + /// public void Recover() { foreach (var group in this.psfGroups.Values) @@ -578,6 +868,10 @@ public void Recover() #region Log Operations + /// + /// Flush logs for all s until their current tail (records are still retained in memory) + /// + /// Synchronous wait for operation to complete public void FlushLogs(bool wait) { foreach (var group in this.psfGroups.Values) @@ -585,7 +879,7 @@ public void FlushLogs(bool wait) } /// - /// Flush log and evict all records from memory + /// Flush logs for all s and evict all records from memory /// /// Synchronous wait for operation to complete /// When wait is false, this tells whether the full eviction was successfully registered with FASTER @@ -602,7 +896,7 @@ public bool FlushAndEvictLogs(bool wait) } /// - /// Delete log entirely from memory. Cannot allocate on the log + /// Delete logs for all s entirely from memory. Cannot allocate on the log /// after this point. This is a synchronous operation. /// public void DisposeLogsFromMemory() diff --git a/cs/src/core/Index/PSF/PSFRegistrationSettings.cs b/cs/src/core/Index/PSF/PSFRegistrationSettings.cs index c10e96bf1..562100a72 100644 --- a/cs/src/core/Index/PSF/PSFRegistrationSettings.cs +++ b/cs/src/core/Index/PSF/PSFRegistrationSettings.cs @@ -10,7 +10,7 @@ public class PSFRegistrationSettings { /// /// When registring new PSFs over an existing store, this is the logicalAddress in the primary - /// FasterKV at which indexing will be started. + /// FasterKV at which indexing will be started. TODO: LogicalAddress is FasterKV-specific; revisit when indexing existing records. /// public long IndexFromAddress = Constants.kInvalidAddress; @@ -47,13 +47,13 @@ public class PSFRegistrationSettings public bool ThreadAffinitized; /// - /// The size of the first IPU Cache; inserts are done into this cache only. If zero, no caching is done. + /// The size of the first IPU Cache; inserts are done into this cache only. If zero, no caching is done. // TODOCache /// public long IPU1CacheSize = 0; /// /// The size of the second IPU Cache; inserts are not done into this cache, so more distant records - /// are likelier to remain. If this is nonzero, must also be nonzero. + /// are likelier to remain. If this is nonzero, must also be nonzero. // TODOCache /// public long IPU2CacheSize = 0; } diff --git a/cs/src/core/Index/PSF/README.md b/cs/src/core/Index/PSF/README.md new file mode 100644 index 000000000..cac765cd2 --- /dev/null +++ b/cs/src/core/Index/PSF/README.md @@ -0,0 +1,136 @@ +Faster Predicate Subset Functions (PSFs) +---------------------------------------- + +PSFs are based upon the [FishStore](https://github.com/microsoft/FishStore) prototype. PSFs function as secondary indexes, allowing an application to query on keys other than the single key used by the FasterKV ("Faster Key/Value store"). + + + + + +- [Overview](#overview) + * [FasterKV: Primary Index With Unique Keys](#fasterkv-primary-index-with-unique-keys) + * [PSFs: Secondary Indexes With Nonunique Keys](#psfs-secondary-indexes-with-nonunique-keys) + + [Defining Secondary Indexes](#defining-secondary-indexes) + + [Updating Secondary Indexes](#updating-secondary-indexes) + + [Querying Secondary Indexes](#querying-secondary-indexes) + + [Limitations](#limitations) + - [No Range Indexes](#no-range-indexes) + - [Fixed-Length `TPSFKey`](#fixed-length-tpsfkey) +- [Public API](#public-api) + * [The Core PSF API](#the-core-psf-api) + + [`PSFManager`](#psfmanager) + + [`IPSF`](#ipsf) + * [The FasterKV PSF API](#the-fasterkv-psf-api) + + [Registering PSFs](#registering-psfs) + + [Querying PSFs](#querying-psfs) + + [The FasterPSFSample playground app](#the-fasterpsfsample-playground-app) + + [Registering PSFs on FasterKV](#registering-psfs-on-fasterkv) + - [Registering PSFs on `Restore`](#registering-psfs-on-restore) + * [Querying PSFs on Session](#querying-psfs-on-session) +- [Code internals](#code-internals) + * [KeyPointer](#keypointer) + + + +# Overview +PSFs are "Predicate Subset Functions"; they allow defining predicates that records will match, possibly non-uniquely, for secondary indexing. PSFs are designed to be used by any data provider. Currently there is only an implementation using FasterKV as the provider, so this document will mostly reference the implementation of them as a secondary index (using "secondary FasterKVs") for a primary FasterKV store, with occasional commentary on other possible stores. + +## FasterKV: Primary Index With Unique Keys +FasterKV is essentially a hash table; as such, it has a single primary key for a given record, and there are zero or one records available for a given key. + +- An Upsert (insert or blind update) will replace an identical key, or insert a new record if an exact key match is not found +- An RMW (Read-Modify-Write) will find an exact key match and update the record, or insert a new record if an identical key match is not found +- A Read will find either a single record matching the key, or no records + +The FasterKV Key and Value may be blittable, variable length, or objects. + +## PSFs: Secondary Indexes With Nonunique Keys +PSFs implement secondary indexes by allowing the user to register a delegate that returns an alternate key for the record. For example, a record might be { Id: 42, Species: "cat" }. The "primary index" is the key inserted into the primary FasterKV; in this example it is Id, and there will be only one record with an Id of 42. A PSF might be defined for such records that returns the Species property (in C# terms, a simple lambda such as "() => this.Species;"). This return is nullable, reflecting the "predicate" terminology, which means that the record may or may not "match" the PSF; for example, a record with no pets would return null, and the record would not be stored for that PSF. This design allows zero, one, or more records to be stored for a single PSF key, entirely depending on the PSF definition. + +### Defining Secondary Indexes +PSF definition is done using the [`RegisterPSF` APIs](#registering-psfs-on-fasterkv) on the [`PSFManager`](./PSFManager.cs) class; for the FasterKV provider, a thin wrapper over these exists on the `IFasterKV` interface and is implemented by FasterKV. Each `RegisterPSF` call creates a [`PSFGroup`](./PSFGroup.cs) internally; the [`PSFGroup`](./PSFGroup.cs) contains its own FasterKV instance, and all specified PSF keys are linked in chains within that FasterKV instance. More details of this are shown below. Allowing [`PSFGroup`](./PSFGroup.cs)s has the following advantages: +- A single hashtable can be used for all PSFs, using the ordinal of the PSF as part of the hashing logic. This can save space. +- PSFs should be registered in groups where it is expected that a record will match all or none of the PSFs (that is, if a record results in a non-null key forone PSF in the group, it results in a non-null key for all PSFs in the group, and if a record results in a null key for one PSF in the group, it results in a null key for all PSFs in the group). This saves some overhead in processing variable-length composite keys in the secondary FasterKV; this [KeyPointer](#keypointer) structure is described more fully below. +- All PSFs in a [`PSFGroup`](./PSFGroup.cs) have the same `TPSFKey` type, but different groups can have different `TPSFKey` types. + +Internally, PSFs are implemented using secondary FasterKV instances with separate Key and Value types, as described in the following sections. + +### Updating Secondary Indexes +When a record is inserted into FasterKV (via Upsert or RMW), it is inserted at a unique "logical address" (essentially a page/offset combination). PSFs implement secondary indexes in Faster by allowing the user to register a delegate that returns an alternate key for the record; then the logical address of an insert (termed a RecordId; note that this is *not* the actual record value) is the value that is inserted into the secondary FasterKV instance using the alternate key. The distinction between the Key and Value defined for the primary FasterKV and the secondary Key and Value (the RecordId) is critical; the secondary FasterKV has no idea of the primary datastore's Key and Value definitions. + +Unlike the primary FasterKV's keys, the PSF keys may return multiple records. To query a PSF, the user passes a value for the alternate key; all logical addresses that were inserted with that key are returned, and then the primary FasterKV instance retrieves the actual records at those logical addresses. Whereas the primary FasterKV returns only a single record (if found) via the IFunctions callback implementation supplied by the client, PSF queries return an `IEnumerable` or `IAsyncEnumerable`. + +The logical address as the RecordId is specific the the FasterKV's use of PSFs; other data provider could provide any other record identifier that fits with their design. + +PSF updating is done by the Primary FasterKV's Upsert, RMW, or Delete operations, which call the Upsert, Update, or Delete methods on the [`PSFManager`](./PSFManager.cs). + +### Querying Secondary Indexes +The `ClientSession` object contains several overloads of `QueryPSF`, taking various combinations of [`IPSF`](./IPSF.cs), `TPSFKey` types and individual keys, and `matchPredicate`. + +The simplest query takes only a single [`IPSF`](./IPSF.cs) and `TPSFKey` key instance, returning all records that match that key for that [`IPSF`](./IPSF.cs). More complicated forms allow specifying multiple [`IPSF`](./IPSF.cs)s, multiple keys per [`IPSF`](./IPSF.cs), and multiple `TPSFKey` types (each with multiple [`IPSF`](./IPSF.cs)s, each with multiple keys). + +Because [`PSFGroup`](./PSFGroup.cs)s store `TRecordId`s, the [`PSFManager`](./PSFManager.cs) client (that is, the data provider, such as a primary FasterKV) must wrap the `QueryPSF` call with its own translation of the `TRecordId` to the actual data record; see `CreateProviderData` in `FasterPSFSessionOperations.cs`. + +The [PSFQuery API](#querying-psfs-on-session) is described in detail below. + +### Limitations +The current implementation of PSFs has some limitations. + +#### No Range Indexes +Because PSFs use hash indexes (and conceptually store multiple records for a given key as that key's collision chain), we have only equality comparisons, not ranges. This can be worked around in some cases by specifying "binned" keys; for example, a date range can be represented as bins of one minute each. In this case, the query will pass an enumeration of keys and all records for all keys will be queried; the caller must post-process each record to ensure it is within the desired range. The caller must decide the bin size, trading off the inefficiency of multiple key enumerations with the inefficiency of returning unwanted values (including the lookuup of the logical address in the primary FasterKV). + +#### Fixed-Length `TPSFKey` +PSFs currently use fixed-length keys; the `TPSFKey` type returned by a PSF execution has a type constraint of being `struct`. It must be a blittable type; the PSF API does not provide for `IVariableLengthStruct` or `SerializerSettings`, nor does it accept strings as keys. Rather than passing a string, the caller must pass some sort of string identifier, such as a hash (but do not use string.GetHashCode() for this, because it is AppDomain-specific (and also dependent on .NET version)). In this case, the hashcode becomes a "bin" of all strings (or string prefixes) corresponding to that hash code. + +# Public API +This section discusses the PSF API in more detail. There are two levels: The interface to PSFs themselves, which is [`PSFManager`](./PSFManager.cs), and how FasterKV is a client of PSFs (as well as providing code for the implementation). + +## The Core PSF API +This section describes the core PSF API on the [`PSFManager`](./PSFManager.cs) class. For examples of its use, see [The FasterKV PSF API](#the-fasterkv-psf-api), which wraps the core PSF API. + +### [`PSFManager`](./PSFManager.cs) +As discussed above, the PSF API is intended to be used by any data provider needing a hash-based index that is capable of storing a `TRecordId` from which the provider can extract its full record. However, PSF update operations must be able to execute the PSF, which requires knowledge of the provider's Key and Value types. Therefore, [`PSFManager`](./PSFManager.cs) has two generic types, both of which are opaque to PSFs: +- `TProviderData`, which is the data passed to PSF execution (the PSF must, of course, know how to operate on the provider data and form a `TPSFKey` key from it) +- `TRecordId`, which is the record identifier stored as the Value in the secondary FasterKV. + +The `TRecordId` has a type constraint of being `struct`; it must be blittable. + +### [`IPSF`](./IPSF.cs) + +## The FasterKV PSF API + +### Registering PSFs + +### Querying PSFs + +### The FasterPSFSample playground app +The [FasterPSFSample](../../../../playground/FasterPSFSample/FasterPSFSample.cs) app demonstrates registering and querying PSFs. + +### Registering PSFs on FasterKV +[Defining Secondary Indexes](#defining-secondary-indexes) provides an overview of registering PSFs. For the FasterKV provider, this is done on the [IFasterKV](../Interfaces/IFasterKV.cs) interface, because it does not do any session operations itself. TODO revisit for indexing of existing records. + +[FPSF.cs](../../../../playground/FasterPSFSample/FPSF.cs) illustrates registering the PSFs. There are several overloads, depending on the number of PSFs per group and whether the PSF can be defined by a simple lambda vs. a more complex definition (which requires an implementation of [`IPSFDefinition`](./IPSFDefinition.cs)). + +[`PSFGroup`](./PSFGroup.cs)s are not visible to the client; they are entirely internal. The return from a `RegisterPSF` call is a vector of [`IPSF`](./IPSF.cs), and various combinations of [`IPSF`](./IPSF.cs) and `TPSFKey` values are passed the the [`QueryPSF` API](#querying-secondary-indexes). + +Each [`PSFGroup`](./PSFGroup.cs)s contains its own internal "secondary" FasterKV instance, including a hash table (shared by all PSFs, including the PSF ordinal in the hash operation) and log. Because this contains hashed records for all PSFs in the group, PSFs cannot be modified once created, nor can they be added to or removed from the group individually. + +#### Registering PSFs on `Restore` +[IFasterKV](../Interfaces/IFasterKV.cs) provides a `GetRegisteredPSFNames` method that returns the names of all PSFs that were registered. Another provider would have to expose similar functionality. At `Restore` time, before any operations are done on the Primary FasterKV, the aplication must call `RegisterPSF` on those names for those groups; it *must not* change a group definition by adding, removing, or supplying a different name or functionality (lambda or definition) for a PSF in the group; doing so will break access to existing records in the group. + +If an application creates a new version of a PSF, it should encode the version information into the PSF's name, e.g. "Dog v1.1". The application must keep track internally of all PSF names and functionality (lambda or definition) for any groups it has created. + +Dropping a group is done by omitting it from the `RegisterPSF` calls done at `Restore` time. This is the only supported versioning mechanism: "drop" a group (by not registering it) and then create a new group with the updated definitions (and possibly changed PSF membership). + +## Querying PSFs on Session +[Querying Secondary Indexes](#querying-secondary-indexes) provides an overview of querying PSFs. For the FasterKV provider, these are done on the [ClientSession](FasterPSFSessionOperations.cs) class, because they must enter the session lock. + +# Code internals +This section presents a high-level overview of the PSF internal design and implementation. + +## KeyPointer +TODO + +Notes: +- generics don't have type constraint for ": nullable" (but they can for notnull) +- \ No newline at end of file From ffbad7bba664ecb59480250a9055cc417441ffbd Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Thu, 3 Sep 2020 21:42:42 -0700 Subject: [PATCH 18/19] Changes to PSFs to match most recent upstream (removal of new() constraint, etc.) --- cs/playground/FasterPSFSample/FPSF.cs | 5 ++- cs/playground/FasterPSFSample/NoSerializer.cs | 4 +-- cs/playground/FasterPSFSample/ObjectOrders.cs | 5 +-- cs/src/core/ClientSession/ClientSession.cs | 34 ++++--------------- cs/src/core/Index/PSF/CompositeKey.cs | 1 - .../core/Index/PSF/FasterKVPSFDefinition.cs | 2 -- cs/src/core/Index/PSF/FasterKVProviderData.cs | 2 -- .../Index/PSF/FasterPSFContextOperations.cs | 2 -- cs/src/core/Index/PSF/FasterPSFImpl.cs | 2 -- .../core/Index/PSF/FasterPSFLogOperations.cs | 2 -- .../core/Index/PSF/FasterPSFRegistration.cs | 2 -- .../Index/PSF/FasterPSFSessionOperations.cs | 14 ++++---- cs/src/core/Index/PSF/KeyAccessor.cs | 1 - cs/src/core/Index/PSF/PSFChangeTracker.cs | 2 +- cs/src/core/Index/PSF/PSFFunctions.cs | 24 ++++++------- cs/src/core/Index/PSF/PSFInput.cs | 4 +-- cs/src/core/Index/PSF/PSFOutput.cs | 2 -- cs/src/core/Index/PSF/PSFUpdateArgs.cs | 2 -- 18 files changed, 32 insertions(+), 78 deletions(-) diff --git a/cs/playground/FasterPSFSample/FPSF.cs b/cs/playground/FasterPSFSample/FPSF.cs index 5968acb9c..cc14a2498 100644 --- a/cs/playground/FasterPSFSample/FPSF.cs +++ b/cs/playground/FasterPSFSample/FPSF.cs @@ -9,9 +9,8 @@ namespace FasterPSFSample { class FPSF - where TValue : IOrders, new() - where TOutput : new() - where TFunctions : IFunctions>, new() + where TValue : IOrders + where TFunctions : IFunctions> where TSerializer : BinaryObjectSerializer, new() { internal IFasterKV FasterKV { get; set; } diff --git a/cs/playground/FasterPSFSample/NoSerializer.cs b/cs/playground/FasterPSFSample/NoSerializer.cs index 508fa57f3..403bde36c 100644 --- a/cs/playground/FasterPSFSample/NoSerializer.cs +++ b/cs/playground/FasterPSFSample/NoSerializer.cs @@ -8,10 +8,10 @@ namespace FasterPSFSample { public class NoSerializer : BinaryObjectSerializer { - public override void Deserialize(ref BlittableOrders obj) + public override void Deserialize(out BlittableOrders obj) => throw new NotImplementedException("NoSerializer should not be instantiated"); - public override void Serialize(ref BlittableOrders obj) + public override void Serialize(in BlittableOrders obj) => throw new NotImplementedException("NoSerializer should not be instantiated"); } } diff --git a/cs/playground/FasterPSFSample/ObjectOrders.cs b/cs/playground/FasterPSFSample/ObjectOrders.cs index c0693546d..5f0ffdb1e 100644 --- a/cs/playground/FasterPSFSample/ObjectOrders.cs +++ b/cs/playground/FasterPSFSample/ObjectOrders.cs @@ -28,14 +28,15 @@ public class ObjectOrders : IOrders public class Serializer : BinaryObjectSerializer { - public override void Deserialize(ref ObjectOrders obj) + public override void Deserialize(out ObjectOrders obj) { + obj = new ObjectOrders(); obj.values = new int[numValues]; for (var ii = 0; ii < obj.values.Length; ++ii) obj.values[ii] = reader.ReadInt32(); } - public override void Serialize(ref ObjectOrders obj) + public override void Serialize(in ObjectOrders obj) { for (var ii = 0; ii < obj.values.Length; ++ii) writer.Write(obj.values[ii]); diff --git a/cs/src/core/ClientSession/ClientSession.cs b/cs/src/core/ClientSession/ClientSession.cs index 862367732..ec505544d 100644 --- a/cs/src/core/ClientSession/ClientSession.cs +++ b/cs/src/core/ClientSession/ClientSession.cs @@ -250,18 +250,7 @@ public Status RMW(ref Key key, ref Input input, Context userContext, long serial /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref Key key, ref Input input) - { - if (SupportAsync) UnsafeResumeThread(); - try - { - return fht.ContextRMW(ref key, ref input, default, FasterSession, 0, ctx); - } - finally - { - if (SupportAsync) UnsafeSuspendThread(); - } - } + public Status RMW(ref Key key, ref Input input) => this.RMW(ref key, ref input, default, 0); /// /// Async RMW operation @@ -315,6 +304,10 @@ public Status Delete(ref Key key, Context userContext, long serialNo) { if (SupportAsync) UnsafeSuspendThread(); } + + return status == Status.OK && this.fht.PSFManager.HasPSFs + ? this.fht.PSFManager.Delete(updateArgs.ChangeTracker) + : status; } /// @@ -323,22 +316,7 @@ public Status Delete(ref Key key, Context userContext, long serialNo) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref Key key) - { - if (SupportAsync) UnsafeResumeThread(); - try - { - return fht.ContextDelete(ref key, default, FasterSession, 0, ctx); - } - finally - { - if (SupportAsync) UnsafeSuspendThread(); - } - - return status == Status.OK && this.fht.PSFManager.HasPSFs - ? this.fht.PSFManager.Delete(updateArgs.ChangeTracker) - : status; - } + public Status Delete(ref Key key) => this.Delete(ref key, default, 0); /// /// Async delete operation diff --git a/cs/src/core/Index/PSF/CompositeKey.cs b/cs/src/core/Index/PSF/CompositeKey.cs index bc486ff16..fe0b31aec 100644 --- a/cs/src/core/Index/PSF/CompositeKey.cs +++ b/cs/src/core/Index/PSF/CompositeKey.cs @@ -10,7 +10,6 @@ namespace FASTER.core /// /// public unsafe struct CompositeKey - where TPSFKey : new() { // This class is essentially a "reinterpret_cast*>" implementation; there are no data members. diff --git a/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs b/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs index 20d711e34..59acda16f 100644 --- a/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs +++ b/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs @@ -13,8 +13,6 @@ namespace FASTER.core /// The type of the key returned by the Predicate and store in the secondary /// (PSF-implementing) FasterKV instances public class FasterKVPSFDefinition : IPSFDefinition, TPSFKey> - where TKVKey : new() - where TKVValue : new() where TPSFKey : struct { /// diff --git a/cs/src/core/Index/PSF/FasterKVProviderData.cs b/cs/src/core/Index/PSF/FasterKVProviderData.cs index 1cbb35281..35ccf8283 100644 --- a/cs/src/core/Index/PSF/FasterKVProviderData.cs +++ b/cs/src/core/Index/PSF/FasterKVProviderData.cs @@ -16,8 +16,6 @@ namespace FASTER.core /// FasterKV instances, and the actual and /// types. public class FasterKVProviderData : IDisposable - where TKVKey : new() - where TKVValue : new() { // C# doesn't allow ref fields and even if it did, if the client held the FasterKVProviderData // past the ref lifetime, bad things would happen when accessing the ref key/value. diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs index 109d683f3..14fdc7fe3 100644 --- a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -8,8 +8,6 @@ namespace FASTER.core { public partial class FasterKV : FasterBase, IFasterKV - where Key : new() - where Value : new() { [MethodImpl(MethodImplOptions.AggressiveInlining)] internal Status ContextPsfReadKey(ref Key key, ref Input input, ref Output output, ref Context context, diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs index 3bbb883be..e917b2bb7 100644 --- a/cs/src/core/Index/PSF/FasterPSFImpl.cs +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -15,8 +15,6 @@ namespace FASTER.core // PSF-related internal function implementations for FasterKV; these correspond to the similarly-named // functions in FasterImpl.cs. public unsafe partial class FasterKV : FasterBase, IFasterKV - where Key : new() - where Value : new() { internal KeyAccessor PsfKeyAccessor => this.hlog.PsfKeyAccessor; diff --git a/cs/src/core/Index/PSF/FasterPSFLogOperations.cs b/cs/src/core/Index/PSF/FasterPSFLogOperations.cs index 6420ed17c..dfb2fa3de 100644 --- a/cs/src/core/Index/PSF/FasterPSFLogOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFLogOperations.cs @@ -8,8 +8,6 @@ namespace FASTER.core { public partial class FasterKV : FasterBase, IFasterKV - where Key : new() - where Value : new() { /// public void FlushPSFLogs(bool wait) => this.PSFManager.FlushLogs(wait); diff --git a/cs/src/core/Index/PSF/FasterPSFRegistration.cs b/cs/src/core/Index/PSF/FasterPSFRegistration.cs index 5c7261dc0..69ba59afd 100644 --- a/cs/src/core/Index/PSF/FasterPSFRegistration.cs +++ b/cs/src/core/Index/PSF/FasterPSFRegistration.cs @@ -10,8 +10,6 @@ namespace FASTER.core { // PSF-related functions for FasterKV public partial class FasterKV : FasterBase, IFasterKV - where Key : new() - where Value : new() { internal PSFManager, long> PSFManager { get; private set; } diff --git a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs index b4bc167f0..aa82d5265 100644 --- a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs +++ b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs @@ -9,18 +9,16 @@ namespace FASTER.core { public sealed partial class ClientSession : IClientSession, IDisposable - where Key : new() - where Value : new() where Functions : IFunctions { #region PSF calls for Secondary FasterKV // This value is created within the Primary FKV session. - Lazy, PSFOutputPrimaryReadAddress, PSFContext, PSFPrimaryFunctions>> psfLookupRecordIdSession; + Lazy, PSFContext, PSFPrimaryFunctions>> psfLookupRecordIdSession; internal void CreateLazyPsfSessionWrapper() { - this.psfLookupRecordIdSession = new Lazy, PSFOutputPrimaryReadAddress, PSFContext, PSFPrimaryFunctions>>( - () => this.fht.NewSession, PSFOutputPrimaryReadAddress, PSFContext, PSFPrimaryFunctions>( + this.psfLookupRecordIdSession = new Lazy, PSFContext, PSFPrimaryFunctions>>( + () => this.fht.NewSession, PSFContext, PSFPrimaryFunctions>( new PSFPrimaryFunctions())); } @@ -30,7 +28,7 @@ internal void DisposeLazyPsfSessionWrapper() this.psfLookupRecordIdSession.Value.Dispose(); } - private ClientSession, PSFOutputPrimaryReadAddress, PSFContext, PSFPrimaryFunctions> GetPsfLookupRecordSession() + private ClientSession, PSFContext, PSFPrimaryFunctions> GetPsfLookupRecordSession() => this.psfLookupRecordIdSession.Value; internal Status PsfInsert(ref Key key, ref Value value, ref Input input, ref Context context, long serialNo) @@ -128,7 +126,7 @@ internal Status CreateProviderData(long logicalAddress, ConcurrentQueue(this.fht.hlog, providerDatas); - var input = new PSFInputPrimaryReadAddress(logicalAddress); + var input = new PSFInputPrimaryReadAddress(logicalAddress); var session = this.GetPsfLookupRecordSession(); var context = new PSFContext { Functions = session.functions }; return session.PsfReadAddress(ref input, ref output, ref context, this.ctx.serialNum + 1); @@ -166,7 +164,7 @@ internal async ValueTask> CreateProviderDataAsy { // Looks up logicalAddress in the primary FasterKV var output = new PSFOutputPrimaryReadAddress(this.fht.hlog, providerDatas); - var input = new PSFInputPrimaryReadAddress(logicalAddress); + var input = new PSFInputPrimaryReadAddress(logicalAddress); var session = this.GetPsfLookupRecordSession(); var context = new PSFContext { Functions = session.functions }; var readAsyncResult = await session.PsfReadAddressAsync(ref input, ref output, ref context, this.ctx.serialNum + 1, querySettings); diff --git a/cs/src/core/Index/PSF/KeyAccessor.cs b/cs/src/core/Index/PSF/KeyAccessor.cs index 24fb13629..c91d55737 100644 --- a/cs/src/core/Index/PSF/KeyAccessor.cs +++ b/cs/src/core/Index/PSF/KeyAccessor.cs @@ -12,7 +12,6 @@ namespace FASTER.core /// /// The type of the Key returned by a PSF function internal unsafe class KeyAccessor - where TPSFKey : new() { private readonly IFasterEqualityComparer userComparer; diff --git a/cs/src/core/Index/PSF/PSFChangeTracker.cs b/cs/src/core/Index/PSF/PSFChangeTracker.cs index 5b6d5d886..33fcd3876 100644 --- a/cs/src/core/Index/PSF/PSFChangeTracker.cs +++ b/cs/src/core/Index/PSF/PSFChangeTracker.cs @@ -19,7 +19,7 @@ public enum UpdateOperation } public unsafe class PSFChangeTracker : IDisposable - where TRecordId : new() + // TODO: where TRecordId : struct { #region Data API internal TProviderData BeforeData { get; private set; } diff --git a/cs/src/core/Index/PSF/PSFFunctions.cs b/cs/src/core/Index/PSF/PSFFunctions.cs index 9194f2209..e9e2ee248 100644 --- a/cs/src/core/Index/PSF/PSFFunctions.cs +++ b/cs/src/core/Index/PSF/PSFFunctions.cs @@ -39,20 +39,18 @@ internal interface IPSFFunctions : IPSFFunctions /// /// The type of the user key in the primary Faster KV /// The type of the user value in the primary Faster KV - internal class PSFPrimaryFunctions : StubbedFunctions, PSFOutputPrimaryReadAddress>, - IPSFFunctions, PSFOutputPrimaryReadAddress> - where TKVKey : new() - where TKVValue : new() + internal class PSFPrimaryFunctions : StubbedFunctions>, + IPSFFunctions> { - public long GroupId(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + public long GroupId(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); - public bool IsDelete(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); - public bool SetDelete(ref PSFInputPrimaryReadAddress input, bool value) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + public bool IsDelete(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + public bool SetDelete(ref PSFInputPrimaryReadAddress input, bool value) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long ReadLogicalAddress(ref PSFInputPrimaryReadAddress input) => input.ReadLogicalAddress; + public long ReadLogicalAddress(ref PSFInputPrimaryReadAddress input) => input.ReadLogicalAddress; - public ref TKVKey QueryKeyRef(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + public ref TKVKey QueryKeyRef(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); [MethodImpl(MethodImplOptions.AggressiveInlining)] public PSFOperationStatus VisitPrimaryReadAddress(ref TKVKey key, ref TKVValue value, ref PSFOutputPrimaryReadAddress output, bool isConcurrent) @@ -63,11 +61,11 @@ public PSFOperationStatus VisitPrimaryReadAddress(ref TKVKey key, ref TKVValue v } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public PSFOperationStatus VisitSecondaryRead(ref TKVValue value, ref PSFInputPrimaryReadAddress input, ref PSFOutputPrimaryReadAddress output, + public PSFOperationStatus VisitSecondaryRead(ref TKVValue value, ref PSFInputPrimaryReadAddress input, ref PSFOutputPrimaryReadAddress output, long physicalAddress, bool tombstone, bool isConcurrent) => throw new PSFInternalErrorException("Cannot call this form of Visit() on the primary FKV"); // TODO review error messages - public PSFOperationStatus VisitSecondaryRead(ref TKVKey key, ref TKVValue value, ref PSFInputPrimaryReadAddress input, ref PSFOutputPrimaryReadAddress output, + public PSFOperationStatus VisitSecondaryRead(ref TKVKey key, ref TKVValue value, ref PSFInputPrimaryReadAddress input, ref PSFOutputPrimaryReadAddress output, bool tombstone, bool isConcurrent) => throw new PSFInternalErrorException("Cannot call this form of Visit() on the primary FKV"); // TODO review error messages } @@ -79,8 +77,8 @@ public PSFOperationStatus VisitSecondaryRead(ref TKVKey key, ref TKVValue value, /// The type of the value public class PSFSecondaryFunctions : StubbedFunctions, PSFOutputSecondary>, IPSFFunctions, PSFOutputSecondary> - where TPSFKey : new() - where TRecordId : new() + where TPSFKey : struct + where TRecordId : struct { public long GroupId(ref PSFInputSecondary input) => input.GroupId; diff --git a/cs/src/core/Index/PSF/PSFInput.cs b/cs/src/core/Index/PSF/PSFInput.cs index 9a8510445..5431ff6e0 100644 --- a/cs/src/core/Index/PSF/PSFInput.cs +++ b/cs/src/core/Index/PSF/PSFInput.cs @@ -12,9 +12,7 @@ namespace FASTER.core /// Input to PsfInternalReadAddress on the primary (stores user values) FasterKV to retrieve the Key and Value /// for a logicalAddress returned from the secondary FasterKV instances. This class is FasterKV-provider-specific. /// - /// The type of the key for user values - public unsafe struct PSFInputPrimaryReadAddress - where TKey : new() + public unsafe struct PSFInputPrimaryReadAddress { internal PSFInputPrimaryReadAddress(long readLA) { diff --git a/cs/src/core/Index/PSF/PSFOutput.cs b/cs/src/core/Index/PSF/PSFOutput.cs index 1724ca73e..516ed366a 100644 --- a/cs/src/core/Index/PSF/PSFOutput.cs +++ b/cs/src/core/Index/PSF/PSFOutput.cs @@ -15,8 +15,6 @@ namespace FASTER.core /// The type of the key for user values /// The type of the user values public unsafe struct PSFOutputPrimaryReadAddress - where TKVKey : new() - where TKVValue : new() { internal readonly AllocatorBase allocator; diff --git a/cs/src/core/Index/PSF/PSFUpdateArgs.cs b/cs/src/core/Index/PSF/PSFUpdateArgs.cs index 48821373c..974ae78f4 100644 --- a/cs/src/core/Index/PSF/PSFUpdateArgs.cs +++ b/cs/src/core/Index/PSF/PSFUpdateArgs.cs @@ -7,8 +7,6 @@ namespace FASTER.core /// Structure passed to and set by Upsert and RMW in the Primary FKV; specific to the FasterKV client. /// internal struct PSFUpdateArgs - where Key : new() - where Value : new() { /// /// Set to the inserted or updated logical address, which is the RecordId for the FasterKV client From 97c9f733b077d411a10ad0c094a7669ae4a1e46a Mon Sep 17 00:00:00 2001 From: TedHartMS <15467143+TedHartMS@users.noreply.github.com> Date: Fri, 4 Sep 2020 12:35:35 -0700 Subject: [PATCH 19/19] Fix release build --- cs/src/core/Index/PSF/KeyAccessor.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cs/src/core/Index/PSF/KeyAccessor.cs b/cs/src/core/Index/PSF/KeyAccessor.cs index c91d55737..615bb3a19 100644 --- a/cs/src/core/Index/PSF/KeyAccessor.cs +++ b/cs/src/core/Index/PSF/KeyAccessor.cs @@ -96,8 +96,7 @@ public long GetKeyAddressFromRecordPhysicalAddress(long physicalAddress, int psf => physicalAddress + RecordInfo.GetLength() + psfOrdinal * this.KeyPointerSize; #endregion Address manipulation -#if DEBUG - public string GetString(ref CompositeKey compositeKey, int psfOrdinal = -1) + internal string GetString(ref CompositeKey compositeKey, int psfOrdinal = -1) { if (psfOrdinal == -1) { @@ -115,8 +114,7 @@ public string GetString(ref CompositeKey compositeKey, int psfOrdinal = return this.GetString(ref this.GetKeyPointerRef(ref compositeKey, psfOrdinal)); } - public string GetString(ref KeyPointer keyPointer) + internal string GetString(ref KeyPointer keyPointer) => $"{{{(keyPointer.IsNull ? "null" : keyPointer.Key.ToString())}}}"; -#endif } } \ No newline at end of file