From b3d7f251612a49c6e012d1444332e15505d4c960 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Mon, 10 Jun 2019 11:52:46 -0700 Subject: [PATCH 01/22] Move recovery final primary setup to a function For both shard and non-sharded cases, we want to correctly setup the primary logs. However, in the sharded case, we call RecoveryAsync for every shard we recover from. We don't want to move the upgraded directory and log file for every shard, so we move this final logic to a function so we can call it once in both sharded / non-sharded cases. --- Ambrosia/Ambrosia/Program.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 15d47ba9..7947c157 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -2261,6 +2261,7 @@ private async Task RecoverOrStartAsync(long checkpointToLoad = -1, Recovering = true; _restartWithRecovery = true; await RecoverAsync(checkpointToLoad, testUpgrade); + await PrepareToBecomePrimaryAsync(); Recovering = false; } else @@ -2328,6 +2329,10 @@ private async Task RecoverAsync(long checkpointToLoad = -1, bool testUpgrade = f } await ReplayAsync(replayStream); } + } + + private async Task PrepareToBecomePrimaryAsync() + { var readVersion = long.Parse(RetrieveServiceInfo(InfoTitle("CurrentVersion"))); if (_currentVersion != readVersion) { From 11da977e29595f4af8f06a454b06190f8795195c Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Tue, 11 Jun 2019 17:30:06 -0700 Subject: [PATCH 02/22] Store shard specific state in a class If we want to perform shard recovery in parallel, we need to ensure each shard has it's own set of variables (ex. does not use the same log file variables). To make this easier to share with the non-sharded case, we move the variables to a class. --- Ambrosia/Ambrosia/Program.cs | 197 +++++++++++++++++++++++------------ 1 file changed, 128 insertions(+), 69 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 7947c157..9ee6e046 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -1169,7 +1169,7 @@ private string RetrieveServiceInfo(string key) // Used to hold the bytes which will go in the log. Note that two streams are passed in. The // log stream must write to durable storage and be flushable, while the second stream initiates // actual action taken after the message has been made durable. - private class Committer + internal class Committer { byte[] _buf; volatile byte[] _bufbak; @@ -1934,6 +1934,48 @@ internal async Task AddInitialRowAsync(FlexReadBuffer serviceInitializationMessa } } + + /** + * This contains information associated with a given machine + **/ + internal class MachineState + { + public MachineState(long shardID) + { + ShardID = shardID; + } + public LogWriter CheckpointWriter { get; set; } + public Committer Committer { get; set; } + public ConcurrentDictionary Inputs { get; set; } + public long LastCommittedCheckpoint { get; set; } + public long LastLogFile { get; set; } + public AARole MyRole { get; set; } + public ConcurrentDictionary Outputs { get; set; } + public long ShardID { get; set; } + } + + internal void LoadAmbrosiaState(MachineState state) + { + state.CheckpointWriter = _checkpointWriter; + state.Committer = _committer; + state.Inputs = _inputs; + state.LastCommittedCheckpoint = _lastCommittedCheckpoint; + state.LastLogFile = _lastLogFile; + state.MyRole = _myRole; + state.Outputs = _outputs; + } + + internal void UpdateAmbrosiaState(MachineState state) + { + _checkpointWriter = state.CheckpointWriter; + _committer = state.Committer; + _inputs = state.Inputs; + _lastCommittedCheckpoint = state.LastCommittedCheckpoint; + _lastLogFile = state.LastLogFile; + _myRole = state.MyRole; + _outputs = state.Outputs; + } + public class AmbrosiaOutput : IAsyncVertexOutputEndpoint { AmbrosiaRuntime myRuntime; @@ -2068,7 +2110,7 @@ public async Task FromStreamAsync(Stream stream, string otherProcess, string oth // true when this service is in an active/active configuration. False if set to single node bool _activeActive; - enum AARole { Primary, Secondary, Checkpointer }; + internal enum AARole { Primary, Secondary, Checkpointer }; AARole _myRole; // Log size at which we start a new log file. This triggers a checkpoint, <= 0 if manual only checkpointing is done long _newLogTriggerSize; @@ -2260,7 +2302,9 @@ private async Task RecoverOrStartAsync(long checkpointToLoad = -1, { Recovering = true; _restartWithRecovery = true; - await RecoverAsync(checkpointToLoad, testUpgrade); + MachineState state = new MachineState(_shardID); + await RecoverAsync(state, checkpointToLoad, testUpgrade); + UpdateAmbrosiaState(state); await PrepareToBecomePrimaryAsync(); Recovering = false; } @@ -2270,64 +2314,69 @@ private async Task RecoverOrStartAsync(long checkpointToLoad = -1, } } - private async Task RecoverAsync(long checkpointToLoad = -1, bool testUpgrade = false) + private async Task RecoverAsync(MachineState state, long checkpointToLoad = -1, bool testUpgrade = false) { if (!_runningRepro) { // We are recovering - find the last committed checkpoint - _lastCommittedCheckpoint = long.Parse(RetrieveServiceInfo(InfoTitle("LastCommittedCheckpoint"))); + state.LastCommittedCheckpoint = long.Parse(RetrieveServiceInfo(InfoTitle("LastCommittedCheckpoint", state.ShardID))); } else { // We are running a repro - _lastCommittedCheckpoint = checkpointToLoad; + state.LastCommittedCheckpoint = checkpointToLoad; } // Start from the log file associated with the last committed checkpoint - _lastLogFile = _lastCommittedCheckpoint; + state.LastLogFile = state.LastCommittedCheckpoint; if (_activeActive) { if (!_runningRepro) { // Determines the role as either secondary or checkpointer. If its a checkpointer, _commitBlobWriter holds the write lock on the last checkpoint - DetermineRole(); + DetermineRole(state); } else { // We are running a repro. Act as a secondary - _myRole = AARole.Secondary; + state.MyRole = AARole.Secondary; } } - using (LogReader checkpointStream = new LogReader(_logFileNameBase + "chkpt" + _lastCommittedCheckpoint.ToString())) + using (LogReader checkpointStream = new LogReader(_logFileNameBase + "chkpt" + state.LastCommittedCheckpoint.ToString())) { // recover the checkpoint - Note that everything except the replay data must have been written successfully or we // won't think we have a valid checkpoint here. Since we can only be the secondary or checkpointer, the committer doesn't write to the replay log // Recover committer - _committer = new Committer(_localServiceSendToStream, _persistLogs, this, -1, checkpointStream); + state.Committer = new Committer(_localServiceSendToStream, _persistLogs, this, -1, checkpointStream); // Recover input connections - _inputs = _inputs.AmbrosiaDeserialize(checkpointStream); + state.Inputs = state.Inputs.AmbrosiaDeserialize(checkpointStream); // Recover output connections - _outputs = _outputs.AmbrosiaDeserialize(checkpointStream, this); - UnbufferNonreplayableCalls(); + state.Outputs = state.Outputs.AmbrosiaDeserialize(checkpointStream, this); + UnbufferNonreplayableCalls(state.Outputs); // Restore new service from checkpoint var serviceCheckpoint = new FlexReadBuffer(); FlexReadBuffer.Deserialize(checkpointStream, serviceCheckpoint); - _committer.SendCheckpointToRecoverFrom(serviceCheckpoint.Buffer, serviceCheckpoint.Length, checkpointStream); + state.Committer.SendCheckpointToRecoverFrom(serviceCheckpoint.Buffer, serviceCheckpoint.Length, checkpointStream); + if (_runningRepro) + { + // For replay, we need _outputs to be set for the local listener to work. + UpdateAmbrosiaState(state); + } } - using (LogReader replayStream = new LogReader(_logFileNameBase + "log" + _lastLogFile.ToString())) + using (LogReader replayStream = new LogReader(_logFileNameBase + "log" + state.LastLogFile.ToString())) { - if (_myRole == AARole.Secondary && !_runningRepro) + if (state.MyRole == AARole.Secondary && !_runningRepro) { // If this is a secondary, set up the detector to detect when this instance becomes the primary - var t = DetectBecomingPrimaryAsync(); + var t = DetectBecomingPrimaryAsync(state); } if (testUpgrade) { // We are actually testing an upgrade. Must upgrade the service before replay - _committer.SendUpgradeRequest(); + state.Committer.SendUpgradeRequest(); } - await ReplayAsync(replayStream); + await ReplayAsync(replayStream, state); } } @@ -2387,10 +2436,9 @@ private async Task StartAsync() (new Task(() => CheckForUpgradeAsync())).Start(); } } - - private void UnbufferNonreplayableCalls() + private void UnbufferNonreplayableCalls(ConcurrentDictionary outputs) { - foreach (var outputRecord in _outputs) + foreach (var outputRecord in outputs) { var newLastSeqNo = outputRecord.Value.BufferedOutput.TrimAndUnbufferNonreplayableCalls(outputRecord.Value.TrimTo, outputRecord.Value.ReplayableTrimTo); if (newLastSeqNo != -1) @@ -2474,12 +2522,16 @@ private LogWriter CreateNextLogFile() return retVal; } - private string InfoTitle(string prefix) + private string InfoTitle(string prefix, long shardID = -1) { var file = prefix; if (_sharded) { - file += _shardID.ToString(); + if (shardID == -1) + { + shardID = _shardID; + } + file += shardID.ToString(); } return file; } @@ -2529,34 +2581,34 @@ private async Task MoveServiceToNextLogFileAsync(bool firstStart = fa //============================================================================================================== // Insance compete over write permission for LOG file & CheckPoint file - private void DetermineRole() + private void DetermineRole(MachineState state) { if (_upgrading) { - _myRole = AARole.Secondary; + state.MyRole = AARole.Secondary; return; } try { // Compete for Checkpoint Write Permission - _checkpointWriter = new LogWriter(_logFileNameBase + "chkpt" + (_lastCommittedCheckpoint).ToString(), 1024 * 1024, 6, true); - _myRole = AARole.Checkpointer; // I'm a checkpointing secondary - var oldCheckpoint = _lastCommittedCheckpoint; - _lastCommittedCheckpoint = long.Parse(RetrieveServiceInfo(InfoTitle("LastCommittedCheckpoint"))); - if (oldCheckpoint != _lastCommittedCheckpoint) + state.CheckpointWriter = new LogWriter(_logFileNameBase + "chkpt" + (state.LastCommittedCheckpoint).ToString(), 1024 * 1024, 6, true); + state.MyRole = AARole.Checkpointer; // I'm a checkpointing secondary + var oldCheckpoint = state.LastCommittedCheckpoint; + state.LastCommittedCheckpoint = long.Parse(RetrieveServiceInfo(InfoTitle("LastCommittedCheckpoint", state.ShardID))); + if (oldCheckpoint != state.LastCommittedCheckpoint) { - _checkpointWriter.Dispose(); + state.CheckpointWriter.Dispose(); throw new Exception("We got a handle on an old checkpoint. The checkpointer was alive when this instance started"); } } catch { - _checkpointWriter = null; - _myRole = AARole.Secondary; // I'm a secondary + state.CheckpointWriter = null; + state.MyRole = AARole.Secondary; // I'm a secondary } } - public async Task DetectBecomingPrimaryAsync() + internal async Task DetectBecomingPrimaryAsync(MachineState state) { // keep trying to take the write permission on LOG file // LOG write permission acquired only in case primary failed (is down) @@ -2564,10 +2616,10 @@ public async Task DetectBecomingPrimaryAsync() { try { - var oldLastLogFile = _lastLogFile; + var oldLastLogFile = state.LastLogFile; // Compete for log write permission - non destructive open for write - open for append var lastLogFileStream = new LogWriter(_logFileNameBase + "log" + (oldLastLogFile).ToString(), 1024 * 1024, 6, true); - if (long.Parse(RetrieveServiceInfo(InfoTitle("LastLogFile"))) != oldLastLogFile) + if (long.Parse(RetrieveServiceInfo(InfoTitle("LastLogFile", state.ShardID))) != oldLastLogFile) { // We got an old log. Try again lastLogFileStream.Dispose(); @@ -2575,7 +2627,7 @@ public async Task DetectBecomingPrimaryAsync() } // We got the lock! Set things up so we let go of the lock at the right moment // But first check if we got the lock because the version changed, in which case, we should commit suicide - var readVersion = long.Parse(RetrieveServiceInfo(InfoTitle("CurrentVersion"))); + var readVersion = long.Parse(RetrieveServiceInfo(InfoTitle("CurrentVersion", state.ShardID))); if (_currentVersion != readVersion) { @@ -2591,10 +2643,11 @@ public async Task DetectBecomingPrimaryAsync() } // Now we can really promote! - await _committer.SleepAsync(); - _committer.SwitchLogStreams(lastLogFileStream); - await _committer.WakeupAsync(); - _myRole = AARole.Primary; // this will stop and break the loop in the function replayInput_Sec() + await state.Committer.SleepAsync(); + state.Committer.SwitchLogStreams(lastLogFileStream); + await state.Committer.WakeupAsync(); + + state.MyRole = AARole.Primary; // this will stop and break the loop in the function replayInput_Sec() Console.WriteLine("\n\nNOW I'm Primary\n\n"); // if we are an upgrader : Time to release the kill file lock and cleanup. Note that since we have the log lock // everyone is prevented from promotion until we succeed or fail. @@ -2619,7 +2672,7 @@ public async Task DetectBecomingPrimaryAsync() } } - private async Task ReplayAsync(LogReader replayStream) + private async Task ReplayAsync(LogReader replayStream, MachineState state) { var tempBuf = new byte[100]; var tempBuf2 = new byte[100]; @@ -2641,7 +2694,7 @@ private async Task ReplayAsync(LogReader replayStream) replayStream.ReadAllRequiredBytes(headerBuf, 0, Committer.HeaderSize); headerBufStream.Position = 0; var commitID = headerBufStream.ReadIntFixed(); - if (commitID != _committer.CommitID) + if (commitID != state.Committer.CommitID) { throw new Exception("Committer didn't match. Must be incomplete record"); } @@ -2649,7 +2702,7 @@ private async Task ReplayAsync(LogReader replayStream) commitSize = headerBufStream.ReadIntFixed(); var checkBytes = headerBufStream.ReadLongFixed(); var writeSeqID = headerBufStream.ReadLongFixed(); - if (writeSeqID != _committer._nextWriteID) + if (writeSeqID != state.Committer._nextWriteID) { throw new Exception("Out of order page. Must be incomplete record"); } @@ -2661,7 +2714,7 @@ private async Task ReplayAsync(LogReader replayStream) } replayStream.Read(tempBuf, 0, commitSize); // Perform integrity check - long checkBytesCalc = _committer.CheckBytes(tempBuf, 0, commitSize); + long checkBytesCalc = state.Committer.CheckBytes(tempBuf, 0, commitSize); if (checkBytesCalc != checkBytes) { throw new Exception("Integrity check failed for page. Must be incomplete record"); @@ -2711,40 +2764,46 @@ private async Task ReplayAsync(LogReader replayStream) if (detectedEOF) { // Move to the next log file for reading only. We may need to take a checkpoint - _lastLogFile++; + state.LastLogFile++; replayStream.Dispose(); - if (!LogWriter.FileExists(_logFileNameBase + "log" + _lastLogFile.ToString())) + if (!LogWriter.FileExists(_logFileNameBase + "log" + state.LastLogFile.ToString())) { - OnError(MissingLog, "Missing log in replay " + _lastLogFile.ToString()); + OnError(MissingLog, "Missing log in replay " + state.LastLogFile.ToString()); } - replayStream = new LogReader(_logFileNameBase + "log" + _lastLogFile.ToString()); - if (_myRole == AARole.Checkpointer) + replayStream = new LogReader(_logFileNameBase + "log" + state.LastLogFile.ToString()); + if (state.MyRole == AARole.Checkpointer) { // take the checkpoint associated with the beginning of the new log - await _committer.SleepAsync(); + // It's currently too disruptive to the code to pass in MachineState to + // CheckpointAsync, so we update the corresponding variables instead. + // This should be fine since the checkpointer should not replay from + // multiple logs in parallel. + UpdateAmbrosiaState(state); + _committer.SleepAsync(); _committer.QuiesceServiceWithSendCheckpointRequest(); await CheckpointAsync(); await _committer.WakeupAsync(); + LoadAmbrosiaState(state); } detectedEOF = false; continue; } - var myRoleBeforeEOLChecking = _myRole; + var myRoleBeforeEOLChecking = state.MyRole; replayStream.Position = logRecordPos; - var newLastLogFile = _lastLogFile; + var newLastLogFile = state.LastLogFile; if (_runningRepro) { - if (LogWriter.FileExists(_logFileNameBase + "log" + (_lastLogFile + 1).ToString())) + if (LogWriter.FileExists(_logFileNameBase + "log" + (state.LastLogFile + 1).ToString())) { // If there is a next file, then move to it - newLastLogFile = _lastLogFile + 1; + newLastLogFile = state.LastLogFile + 1; } } else { - newLastLogFile = long.Parse(RetrieveServiceInfo(InfoTitle("LastLogFile"))); + newLastLogFile = long.Parse(RetrieveServiceInfo(InfoTitle("LastLogFile", state.ShardID))); } - if (newLastLogFile > _lastLogFile) // a new log file has been written + if (newLastLogFile > state.LastLogFile) // a new log file has been written { // Someone started a new log. Try to read the last record again and then move to next file detectedEOF = true; @@ -2781,22 +2840,22 @@ private async Task ReplayAsync(LogReader replayStream) foreach (var kv in committedInputDict) { InputConnectionRecord inputConnectionRecord; - if (!_inputs.TryGetValue(kv.Key, out inputConnectionRecord)) + if (!state.Inputs.TryGetValue(kv.Key, out inputConnectionRecord)) { // Create input record and add it to the dictionary inputConnectionRecord = new InputConnectionRecord(); - _inputs[kv.Key] = inputConnectionRecord; + state.Inputs[kv.Key] = inputConnectionRecord; } inputConnectionRecord.LastProcessedID = kv.Value.First; inputConnectionRecord.LastProcessedReplayableID = kv.Value.Second; OutputConnectionRecord outputConnectionRecord; // this lock prevents conflict with output arriving from the local service during replay - lock (_outputs) + lock (state.Outputs) { - if (!_outputs.TryGetValue(kv.Key, out outputConnectionRecord)) + if (!state.Outputs.TryGetValue(kv.Key, out outputConnectionRecord)) { outputConnectionRecord = new OutputConnectionRecord(this); - _outputs[kv.Key] = outputConnectionRecord; + state.Outputs[kv.Key] = outputConnectionRecord; } } // this lock prevents conflict with output arriving from the local service during replay and ensures maximal cleaning @@ -2818,12 +2877,12 @@ private async Task ReplayAsync(LogReader replayStream) { OutputConnectionRecord outputConnectionRecord; // this lock prevents conflict with output arriving from the local service during replay - lock (_outputs) + lock (state.Outputs) { - if (!_outputs.TryGetValue(kv.Key, out outputConnectionRecord)) + if (!state.Outputs.TryGetValue(kv.Key, out outputConnectionRecord)) { outputConnectionRecord = new OutputConnectionRecord(this); - _outputs[kv.Key] = outputConnectionRecord; + state.Outputs[kv.Key] = outputConnectionRecord; } } // this lock prevents conflict with output arriving from the local service during replay and ensures maximal cleaning @@ -2837,11 +2896,11 @@ private async Task ReplayAsync(LogReader replayStream) // If this is the first replay segment, it invalidates the contents of the committer, which must be cleared. if (!clearedCommitterWrite) { - _committer.ClearNextWrite(); + state.Committer.ClearNextWrite(); clearedCommitterWrite = true; } // bump up the write ID in the committer in preparation for reading or writing the next page - _committer._nextWriteID++; + state.Committer._nextWriteID++; } } From 2c9c6317cc4a098740edaa8da59225e784194852 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Wed, 12 Jun 2019 14:23:40 -0700 Subject: [PATCH 03/22] Service Name: Incorporate shard ID into the service name The service name folder will change depending on the shardID. This will affect where we look for log and checkpoint files. We update all directory and file lookups to use functions that account for the shardID. Note: For the initial self-connections, I use _serviceName and don't consider if it's a shard. This is because the Sample applications are still linked to the old version of Ambrosia. We will change this when we are able to link Samples to the new version. --- Ambrosia/Ambrosia/Program.cs | 144 ++++++++++++++++++++++++----------- 1 file changed, 101 insertions(+), 43 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 9ee6e046..93a4263c 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -2248,8 +2248,8 @@ private void SetupAzureConnections() private void PrepareToRecoverOrStart() { IPAddress localIPAddress = Dns.GetHostEntry("localhost").AddressList[0]; - LogWriter.CreateDirectoryIfNotExists(_serviceLogPath + _serviceName + "_" + _currentVersion); - _logFileNameBase = Path.Combine(_serviceLogPath + _serviceName + "_" + _currentVersion, "server"); + LogWriter.CreateDirectoryIfNotExists(LogDirectory(_currentVersion)); + _logFileNameBase = LogFileNameBase(_currentVersion); SetupLocalServiceStreams(); if (!_runningRepro) { @@ -2342,7 +2342,7 @@ private async Task RecoverAsync(MachineState state, long checkpointToLoad = -1, } } - using (LogReader checkpointStream = new LogReader(_logFileNameBase + "chkpt" + state.LastCommittedCheckpoint.ToString())) + using (LogReader checkpointStream = new LogReader(CheckpointName(state.LastCommittedCheckpoint, -1, state.ShardID))) { // recover the checkpoint - Note that everything except the replay data must have been written successfully or we // won't think we have a valid checkpoint here. Since we can only be the secondary or checkpointer, the committer doesn't write to the replay log @@ -2364,7 +2364,7 @@ private async Task RecoverAsync(MachineState state, long checkpointToLoad = -1, } } - using (LogReader replayStream = new LogReader(_logFileNameBase + "log" + state.LastLogFile.ToString())) + using (LogReader replayStream = new LogReader(LogFileName(state.LastLogFile, -1, state.ShardID))) { if (state.MyRole == AARole.Secondary && !_runningRepro) { @@ -2426,8 +2426,8 @@ private async Task StartAsync() _checkpointWriter = null; _committer = new Committer(_localServiceSendToStream, _persistLogs, this); - Connect(_serviceName, AmbrosiaDataOutputsName, _serviceName, AmbrosiaDataInputsName); - Connect(_serviceName, AmbrosiaControlOutputsName, _serviceName, AmbrosiaControlInputsName); + Connect(ServiceName(), AmbrosiaDataOutputsName, ServiceName(), AmbrosiaDataInputsName); + Connect(ServiceName(), AmbrosiaControlOutputsName, ServiceName(), AmbrosiaControlInputsName); await MoveServiceToNextLogFileAsync(true, true); InsertOrReplaceServiceInfoRecord(InfoTitle("CurrentVersion"), _currentVersion.ToString()); if (_activeActive) @@ -2436,6 +2436,7 @@ private async Task StartAsync() (new Task(() => CheckForUpgradeAsync())).Start(); } } + private void UnbufferNonreplayableCalls(ConcurrentDictionary outputs) { foreach (var outputRecord in outputs) @@ -2450,8 +2451,8 @@ private void UnbufferNonreplayableCalls(ConcurrentDictionary Date: Thu, 13 Jun 2019 16:41:14 -0700 Subject: [PATCH 04/22] Allow shard ID to be passed in as a parameter This makes it easier to switch between sharded and non-sharded scenario. We still need to figure out the best way to pass in the shards to recover from. --- Ambrosia/Ambrosia/Program.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 93a4263c..943c4b53 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -1057,6 +1057,7 @@ public class AmbrosiaRuntimeParams public string storageConnectionString; public long currentVersion; public long upgradeToVersion; + public long shardID; } public static class AmbrosiaRuntimeParms @@ -3854,7 +3855,9 @@ public override void Initialize(object param) p = (AmbrosiaRuntimeParams)xmlSerializer.Deserialize(textReader); } - bool sharded = false; + bool runningRepro = false; + bool sharded = p.shardID > 0; + _shardID = p.shardID; Initialize( p.serviceReceiveFromPort, @@ -3951,6 +3954,10 @@ bool sharded _serviceName = serviceName; _storageConnectionString = storageConnectionString; _sharded = sharded; + if (_sharded) + { + Console.WriteLine("Running instance with shard ID " + _shardID.ToString()); + } _coral = ClientLibrary; Console.WriteLine("Logs directory: {0}", _serviceLogPath); @@ -4014,6 +4021,7 @@ class Program private static long _logTriggerSizeMB = 1000; private static int _currentVersion = 0; private static long _upgradeVersion = -1; + private static long _shardID = -1; static void Main(string[] args) { @@ -4052,6 +4060,7 @@ static void Main(string[] args) param.serviceLogPath = _serviceLogPath; param.AmbrosiaBinariesLocation = _binariesLocation; param.storageConnectionString = Environment.GetEnvironmentVariable("AZURE_STORAGE_CONN_STRING"); + param.shardID = _shardID; try { @@ -4105,6 +4114,7 @@ private static OptionSet ParseOptions(string[] args, out bool shouldShowHelp) { "rp|receivePort=", "The service receive from port [REQUIRED].", rp => _serviceReceiveFromPort = int.Parse(rp) }, { "sp|sendPort=", "The service send to port. [REQUIRED]", sp => _serviceSendToPort = int.Parse(sp) }, { "l|log=", "The service log path.", l => _serviceLogPath = l }, + {"si|shardID=", "The shard ID of the instance", si => _shardID = long.Parse(si) }, }; var helpOption = new OptionSet From 11e54aadb92fc20b12346a02d1e59e70b0d7e713 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Fri, 14 Jun 2019 09:58:40 -0700 Subject: [PATCH 05/22] Add RuntimeChecksOnProcessStart to sharded case For the sharded case, what is checked changes depending on the shard. --- Ambrosia/Ambrosia/Program.cs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 943c4b53..ce0ad2b4 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -2294,10 +2294,12 @@ private async Task RecoverOrStartAsync(long checkpointToLoad = -1, CheckpointingService = false; Recovering = false; PrepareToRecoverOrStart(); + if (!_runningRepro) { RuntimeChecksOnProcessStart(); } + // Determine if we are recovering if (!_createService) { @@ -3876,18 +3878,21 @@ public override void Initialize(object param) ); } - internal void RuntimeChecksOnProcessStart() + internal void RuntimeChecksOnProcessStart(long shardID = -1) { - if (!_createService) + // When we split / merge shards, these checks will not be valid as the logs for the + // new shards will not exist yet. Instead for the sharded case, we do these checks + // when we recover from the set of old shards. + if (!_createService && !_sharded) { long readVersion = -1; try { - readVersion = long.Parse(RetrieveServiceInfo(InfoTitle("CurrentVersion"))); + readVersion = long.Parse(RetrieveServiceInfo(InfoTitle("CurrentVersion", shardID))); } catch { - OnError(VersionMismatch, "Version mismatch on process start: Expected " + _currentVersion + " was: " + RetrieveServiceInfo(InfoTitle("CurrentVersion"))); + OnError(VersionMismatch, "Version mismatch on process start: Expected " + _currentVersion + " was: " + RetrieveServiceInfo(InfoTitle("CurrentVersion", shardID))); } if (_currentVersion != readVersion) { @@ -3895,7 +3900,7 @@ internal void RuntimeChecksOnProcessStart() } if (!_runningRepro) { - if (long.Parse(RetrieveServiceInfo(InfoTitle("LastCommittedCheckpoint"))) < 1) + if (long.Parse(RetrieveServiceInfo(InfoTitle("LastCommittedCheckpoint", shardID))) < 1) { OnError(MissingCheckpoint, "No checkpoint in metadata"); @@ -3905,12 +3910,12 @@ internal void RuntimeChecksOnProcessStart() { OnError(MissingCheckpoint, "No checkpoint/logs directory"); } - var lastCommittedCheckpoint = long.Parse(RetrieveServiceInfo(InfoTitle("LastCommittedCheckpoint"))); - if (!LogWriter.FileExists(CheckpointName(lastCommittedCheckpoint))) + var lastCommittedCheckpoint = long.Parse(RetrieveServiceInfo(InfoTitle("LastCommittedCheckpoint", shardID))); + if (!LogWriter.FileExists(CheckpointName(lastCommittedCheckpoint, -1, shardID))) { OnError(MissingCheckpoint, "Missing checkpoint " + lastCommittedCheckpoint.ToString()); } - if (!LogWriter.FileExists(LogFileName(lastCommittedCheckpoint))) + if (!LogWriter.FileExists(LogFileName(lastCommittedCheckpoint, -1, shardID))) { OnError(MissingLog, "Missing log " + lastCommittedCheckpoint.ToString()); } From 733aead63c76e6be28d5f25ac1cab51d1189703b Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Mon, 17 Jun 2019 15:34:40 -0700 Subject: [PATCH 06/22] Prepare kill logic to handle shard case --- Ambrosia/Ambrosia/Program.cs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index ce0ad2b4..250203af 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -2528,6 +2528,11 @@ private string LogFileName(long logFile, long version = -1, long shardID = -1) return LogFileNameBase(version, shardID) + "log" + logFile.ToString(); } + private string KillFileName(long shardID = -1) + { + return LogFileNameBase(-1, shardID) + "killFile"; + } + private LogWriter CreateNextOldVerLogFile() { if (LogWriter.FileExists(LogFileName(_lastLogFile + 1, _currentVersion))) @@ -2548,9 +2553,14 @@ private LogWriter CreateNextOldVerLogFile() // Used to create a kill file meant to being down primaries and prevent promotion. Promotion prevention // lasts until the returned file handle is released. - private void LockKillFile() + private void LockKillFile(long shardID = -1) + { + _killFileHandle = LockFile(KillFileName(shardID)); + } + + private FileStream LockFile(string file) { - _killFileHandle = new FileStream(_logFileNameBase + "killFile", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read & ~FileShare.Inheritable); + return new FileStream(file, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read & ~FileShare.Inheritable); } private void ReleaseAndTryCleanupKillFile() @@ -2560,7 +2570,7 @@ private void ReleaseAndTryCleanupKillFile() try { // Try to delete the file. Someone may beat us to it. - File.Delete(_logFileNameBase + "killFile"); + File.Delete(KillFileName()); } catch (Exception e) { From 9eb4c5a14f502664a96f833f5c8cf5f6c88e6c96 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Wed, 26 Jun 2019 10:25:27 -0700 Subject: [PATCH 07/22] Shard Hashing: Implement basic hashing logic Add functionality so Immmortal Coordinator determines which shard to send a message. The shard is determined based on a hash of the destination currently. The hash only returns shard 1 for now. --- Ambrosia/Ambrosia/Program.cs | 98 ++++++++++++++++++++++++++++++---- ImmortalCoordinator/Program.cs | 5 +- 2 files changed, 93 insertions(+), 10 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 250203af..deee0f34 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -2062,6 +2062,7 @@ public async Task FromStreamAsync(Stream stream, string otherProcess, string oth bool _sharded; internal bool _createService; long _shardID; + Func _shardLocator; bool _runningRepro; long _currentVersion; long _upgradeToVersion; @@ -2468,17 +2469,30 @@ public CRAErrorCode Connect(string fromProcessName, string fromEndpoint, string return _coral.Connect(fromProcessName, fromEndpoint, toProcessName, toEndpoint); } - private string ServiceName(long shardID = -1) + private string ServiceName(string name = "", long shardID = -1) { + if (name == "") + { + name = _serviceName; + } + if (_sharded) { if (shardID == -1) { shardID = _shardID; } - return _serviceName + "-" + shardID.ToString(); } - return _serviceName; + return GetShardName(name, shardID); + } + + public static string GetShardName(string name, long shardID) + { + if (shardID != -1) + { + name += "-" + shardID.ToString(); + } + return name; } private string RootDirectory(long version = -1) @@ -3083,6 +3097,20 @@ void AttachTo(string destination) } } + private string DestinationShard(string destination, long shardID = -1) + { + if (destination == "") + { + return destination; + } + if (shardID == -1 && _sharded) + { + shardID = _shardLocator(BitConverter.ToInt32(Encoding.UTF8.GetBytes(destination), 0)); + } + + return ServiceName(destination, shardID); + } + private void ProcessSyncLocalMessage(ref FlexReadBuffer localServiceBuffer, FlexReadBuffer batchServiceBuffer) { var sizeBytes = localServiceBuffer.LengthLength; @@ -3108,6 +3136,7 @@ private void ProcessSyncLocalMessage(ref FlexReadBuffer localServiceBuffer, Flex case attachToByte: // Get dest string var destination = Encoding.UTF8.GetString(localServiceBuffer.Buffer, sizeBytes + 1, localServiceBuffer.Length - sizeBytes - 1); + destination = DestinationShard(destination); localServiceBuffer.ResetBuffer(); if (!_runningRepro) @@ -3226,6 +3255,7 @@ private void ProcessRPC(FlexReadBuffer RpcBuffer) Buffer.BlockCopy(RpcBuffer.Buffer, destOffset, _lastShuffleDest, 0, destBytesSize); _lastShuffleDestSize = destBytesSize; destination = Encoding.UTF8.GetString(RpcBuffer.Buffer, destOffset, destBytesSize); + destination = DestinationShard(destination); // locking to avoid conflict with stream reconnection immediately after replay and trim during replay lock (_outputs) { @@ -3857,6 +3887,27 @@ public AmbrosiaRuntime() : base() { } + public long KeyHashToShard(long hash) + { + if (!_sharded) + { + return -1; + } + + if (_serviceName.Contains("client")) + { + foreach (var input in _inputs.Keys) + { + if (input.Contains("server-2")) + { + return 2; + } + } + } + + return 1; + } + public override void Initialize(object param) { // Workaround because of parameter type limitation in CRA @@ -3870,6 +3921,33 @@ public override void Initialize(object param) bool runningRepro = false; bool sharded = p.shardID > 0; _shardID = p.shardID; + _shardLocator = KeyHashToShard; + + Initialize( + p.serviceReceiveFromPort, + p.serviceSendToPort, + p.serviceName, + p.serviceLogPath, + p.createService, + p.pauseAtStart, + p.persistLogs, + p.activeActive, + p.logTriggerSizeMB, + p.storageConnectionString, + p.currentVersion, + p.upgradeToVersion, + runningRepro, + sharded + ); + } + + public void Initialize(object param, long[] oldShards, long shardID, Func shardMap) + { + var p = (AmbrosiaRuntimeParams)param; + bool runningRepro = true; + bool sharded = true; + _shardID = shardID; + _shardLocator = shardMap; Initialize( p.serviceReceiveFromPort, @@ -4058,7 +4136,8 @@ static void Main(string[] args) var client = new CRAClientLibrary(Environment.GetEnvironmentVariable("AZURE_STORAGE_CONN_STRING")); client.DisableArtifactUploading(); - var replicaName = $"{_instanceName}{_replicaNumber}"; + string shardName = AmbrosiaRuntime.GetShardName(_instanceName, _shardID); + var replicaName = $"{shardName}_{_replicaNumber}"; AmbrosiaRuntimeParams param = new AmbrosiaRuntimeParams(); param.createService = _recoveryMode == AmbrosiaRecoveryModes.A ? (bool?)null @@ -4093,14 +4172,15 @@ static void Main(string[] args) serializedParams = textWriter.ToString(); } - if (client.InstantiateVertex(replicaName, param.serviceName, param.AmbrosiaBinariesLocation, serializedParams) != CRAErrorCode.Success) + if (client.InstantiateVertex(replicaName, shardName, param.AmbrosiaBinariesLocation, serializedParams) != CRAErrorCode.Success) { throw new Exception(); } - client.AddEndpoint(param.serviceName, AmbrosiaRuntime.AmbrosiaDataInputsName, true, true); - client.AddEndpoint(param.serviceName, AmbrosiaRuntime.AmbrosiaDataOutputsName, false, true); - client.AddEndpoint(param.serviceName, AmbrosiaRuntime.AmbrosiaControlInputsName, true, true); - client.AddEndpoint(param.serviceName, AmbrosiaRuntime.AmbrosiaControlOutputsName, false, true); + + client.AddEndpoint(shardName, AmbrosiaRuntime.AmbrosiaDataInputsName, true, true); + client.AddEndpoint(shardName, AmbrosiaRuntime.AmbrosiaDataOutputsName, false, true); + client.AddEndpoint(shardName, AmbrosiaRuntime.AmbrosiaControlInputsName, true, true); + client.AddEndpoint(shardName, AmbrosiaRuntime.AmbrosiaControlOutputsName, false, true); } catch (Exception e) { diff --git a/ImmortalCoordinator/Program.cs b/ImmortalCoordinator/Program.cs index e247bc05..48df8cfb 100644 --- a/ImmortalCoordinator/Program.cs +++ b/ImmortalCoordinator/Program.cs @@ -20,12 +20,14 @@ class Program private static string _secureNetworkClassName; private static bool _isActiveActive = false; private static int _replicaNumber = 0; + private static long _shardID = -1; static void Main(string[] args) { ParseAndValidateOptions(args); - var replicaName = $"{_instanceName}{_replicaNumber}"; + string shardName = AmbrosiaRuntime.GetShardName(_instanceName, _shardID); + var replicaName = $"{shardName}_{_replicaNumber}"; if (_ipAddress == null) { @@ -131,6 +133,7 @@ private static OptionSet ParseOptions(string[] args, out bool shouldShowHelp) { "ac|assemblyClass=", "The secure network assembly class.", ac => _secureNetworkClassName = ac }, { "ip|IPAddr=", "Override automatic self IP detection", i => _ipAddress = i }, { "h|help", "show this message and exit", h => showHelp = h != null }, + { "si|shardID=", "The shard ID", si => _shardID = long.Parse(si) }, }; try From cea3aa44404fb882e225331b281fc0eb20257fb8 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Wed, 26 Jun 2019 14:44:41 -0700 Subject: [PATCH 08/22] Setup InputConnection to track ancestors For shard recovery, we need InputConnectionRecords to track the last processed ID and last processed replayable ID of ancestor shards. This map needs to be sent between peers. We only want to send longs instead of strings containing the name of the peer. To make this easier, we add a ShardID field so that we only have to parse the peer name when we make the initial connection. To make sure we don't break serialization when sharding is added, we move the replay serialization logic into a separate class so we can add unit tests. Any other serialization changes affected by shards will be moved into this class as well. --- Ambrosia/Ambrosia.sln | 6 + Ambrosia/Ambrosia/Program.cs | 127 ++++++++++++++++---- Ambrosia/Ambrosia/Serializer.cs | 95 +++++++++++++++ Ambrosia/AmbrosiaTests/AmbrosiaTests.csproj | 27 +++++ Ambrosia/AmbrosiaTests/SerializerTests.cs | 82 +++++++++++++ 5 files changed, 312 insertions(+), 25 deletions(-) create mode 100644 Ambrosia/Ambrosia/Serializer.cs create mode 100644 Ambrosia/AmbrosiaTests/AmbrosiaTests.csproj create mode 100644 Ambrosia/AmbrosiaTests/SerializerTests.cs diff --git a/Ambrosia/Ambrosia.sln b/Ambrosia/Ambrosia.sln index f8e79b57..1704d0d7 100644 --- a/Ambrosia/Ambrosia.sln +++ b/Ambrosia/Ambrosia.sln @@ -20,6 +20,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Ambrosia", "Ambrosia\Ambros {5852AC33-6B01-44F5-BAF3-2AAF796E8449} = {5852AC33-6B01-44F5-BAF3-2AAF796E8449} EndProjectSection EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AmbrosiaTests", "AmbrosiaTests\AmbrosiaTests.csproj", "{4339AF42-4A49-42E9-8571-0DC4CEB0D1CB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -38,6 +40,10 @@ Global {F704AE0A-C37B-4D30-B9ED-0C76C62D66EC}.Debug|x64.Build.0 = Debug|x64 {F704AE0A-C37B-4D30-B9ED-0C76C62D66EC}.Release|x64.ActiveCfg = Release|x64 {F704AE0A-C37B-4D30-B9ED-0C76C62D66EC}.Release|x64.Build.0 = Release|x64 + {4339AF42-4A49-42E9-8571-0DC4CEB0D1CB}.Debug|x64.ActiveCfg = Debug|x64 + {4339AF42-4A49-42E9-8571-0DC4CEB0D1CB}.Debug|x64.Build.0 = Debug|x64 + {4339AF42-4A49-42E9-8571-0DC4CEB0D1CB}.Release|x64.ActiveCfg = Release|x64 + {4339AF42-4A49-42E9-8571-0DC4CEB0D1CB}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index deee0f34..5e1a4ef5 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -138,6 +138,20 @@ internal static void AmbrosiaSerialize(this ConcurrentDictionary AmbrosiaDese newRecord.LastProcessedID = seqNo; seqNo = readFromStream.ReadLongFixed(); newRecord.LastProcessedReplayableID = seqNo; + newRecord.ShardID = readFromStream.ReadLongFixed(); + Console.WriteLine("Deserialize {0} -> Shard ID {1}", myString, newRecord.ShardID); + var ancestorCount = readFromStream.ReadIntFixed(); + for (int j = 0; j < ancestorCount; j++) + { + var peerID = readFromStream.ReadLongFixed(); + var peerCount = readFromStream.ReadLongFixed(); + newRecord.AncestorsToIDs[peerID] = new ConcurrentDictionary>(); + for (int k = 0; k < peerCount; k++) + { + var shardID = readFromStream.ReadLongFixed(); + var lastProcessedID = readFromStream.ReadLongFixed(); + var lastProcessedReplayableID = readFromStream.ReadLongFixed(); + newRecord.AncestorsToIDs[peerID][shardID] = new Tuple(lastProcessedID, lastProcessedReplayableID); + } + } _retVal.TryAdd(myString, newRecord); } return _retVal; @@ -974,17 +1004,20 @@ internal void RebaseSeqNosInBuffer(long commitSeqNo, [DataContract] internal class InputConnectionRecord { + public ConcurrentDictionary>> AncestorsToIDs { get; set; } public NetworkStream DataConnectionStream { get; set; } public NetworkStream ControlConnectionStream { get; set; } [DataMember] public long LastProcessedID { get; set; } [DataMember] public long LastProcessedReplayableID { get; set; } + public long ShardID { get; set; } public InputConnectionRecord() { DataConnectionStream = null; LastProcessedID = 0; LastProcessedReplayableID = 0; + AncestorsToIDs = new ConcurrentDictionary>>(); } } @@ -2486,6 +2519,21 @@ private string ServiceName(string name = "", long shardID = -1) return GetShardName(name, shardID); } + private Tuple ParseServiceName(string service) + { + if (service == "") + { + return new Tuple(_serviceName, _shardID); + } + + var parts = service.Split('-'); + if (parts.Length == 1) + { + return new Tuple(parts[0], -1); + } + return new Tuple(parts[0], long.Parse(parts[1])); + } + public static string GetShardName(string name, long shardID) { if (shardID != -1) @@ -2931,6 +2979,7 @@ private async Task ReplayAsync(LogReader replayStream, MachineState state) { // Create input record and add it to the dictionary inputConnectionRecord = new InputConnectionRecord(); + inputConnectionRecord.ShardID = ParseServiceName(kv.Key).Item2; state.Inputs[kv.Key] = inputConnectionRecord; } inputConnectionRecord.LastProcessedID = kv.Value.First; @@ -3316,23 +3365,23 @@ private void ProcessRPC(FlexReadBuffer RpcBuffer) } } - private async Task ToDataStreamAsync(Stream writeToStream, - string destString, - CancellationToken ct) - + private async Task SyncOutputConnectionAsync(ConcurrentDictionary outputs, + string destString, + long commitSeqNo, + long commitSeqNoReplayable) { OutputConnectionRecord outputConnectionRecord; if (destString.Equals(ServiceName())) { destString = ""; } - lock (_outputs) + lock (outputs) { - if (!_outputs.TryGetValue(destString, out outputConnectionRecord)) + if (!outputs.TryGetValue(destString, out outputConnectionRecord)) { // Set up the output record for the first time and add it to the dictionary outputConnectionRecord = new OutputConnectionRecord(this); - _outputs[destString] = outputConnectionRecord; + outputs[destString] = outputConnectionRecord; Console.WriteLine("Adding output:{0}", destString); } else @@ -3340,20 +3389,14 @@ private async Task ToDataStreamAsync(Stream writeToStream, Console.WriteLine("restoring output:{0}", destString); } } + try { // Reset the output cursor if it exists outputConnectionRecord.BufferedOutput.AcquireTrimLock(2); outputConnectionRecord.placeInOutput = new EventBuffer.BuffersCursor(null, -1, 0); outputConnectionRecord.BufferedOutput.ReleaseTrimLock(); - // Process replay message - var inputFlexBuffer = new FlexReadBuffer(); - await FlexReadBuffer.DeserializeAsync(writeToStream, inputFlexBuffer, ct); - var sizeBytes = inputFlexBuffer.LengthLength; - // Get the seqNo of the replay/filter point - var commitSeqNo = StreamCommunicator.ReadBufferedLong(inputFlexBuffer.Buffer, sizeBytes + 1); - var commitSeqNoReplayable = StreamCommunicator.ReadBufferedLong(inputFlexBuffer.Buffer, sizeBytes + 1 + StreamCommunicator.LongSize(commitSeqNo)); - inputFlexBuffer.ResetBuffer(); + if (outputConnectionRecord.ConnectingAfterRestart) { // We've been through recovery (at least partially), and have scrubbed all ephemeral calls. Must now rebase @@ -3386,7 +3429,40 @@ private async Task ToDataStreamAsync(Stream writeToStream, } } outputConnectionRecord.LastSeqSentToReceiver = commitSeqNo - 1; + } + catch (Exception e) + { + // Cleanup held locks if necessary + await Task.Yield(); + var lockVal = outputConnectionRecord.BufferedOutput.ReadTrimLock(); + if (lockVal == 1 || lockVal == 2) + { + outputConnectionRecord.BufferedOutput.ReleaseTrimLock(); + } + var bufferLockVal = outputConnectionRecord.BufferedOutput.ReadAppendLock(); + if (bufferLockVal == 2) + { + outputConnectionRecord.BufferedOutput.ReleaseAppendLock(); + } + throw e; + } + return outputConnectionRecord; + } + + private async Task ToDataStreamAsync(Stream writeToStream, + string destString, + CancellationToken ct) + { + // Process replay message + var result = await Serializer.DeserializeReplayMessageAsync(writeToStream, ct); + // Get the seqNo of the replay/filter point + var commitSeqNo = result.Item1; + var commitSeqNoReplayable = result.Item2; + OutputConnectionRecord outputConnectionRecord = await SyncOutputConnectionAsync(_outputs, destString, commitSeqNo, commitSeqNoReplayable); + + try + { // Enqueue a replay send if (outputConnectionRecord._sendsEnqueued == 0) { @@ -3530,18 +3606,17 @@ private async Task ToControlStreamAsync(Stream writeToStream, } private async Task SendReplayMessageAsync(Stream sendToStream, - long lastProcessedID, - long lastProcessedReplayableID, + InputConnectionRecord input, CancellationToken ct) { // Send FilterTo message to the destination command stream - // Write message size - sendToStream.WriteInt(1 + StreamCommunicator.LongSize(lastProcessedID) + StreamCommunicator.LongSize(lastProcessedReplayableID)); - // Write message type - sendToStream.WriteByte(replayFromByte); - // Write the output filter seqNo for the other side - sendToStream.WriteLong(lastProcessedID); - sendToStream.WriteLong(lastProcessedReplayableID); + Serializer.SerializeReplayMessage( + sendToStream, + replayFromByte, + input.LastProcessedID + 1, + input.LastProcessedReplayableID + 1, + input.AncestorsToIDs + ); await sendToStream.FlushAsync(ct); } @@ -3573,6 +3648,7 @@ private async Task FromDataStreamAsync(Stream readFromStream, { // Create input record and add it to the dictionary inputConnectionRecord = new InputConnectionRecord(); + inputConnectionRecord.ShardID = ParseServiceName(sourceString).Item2; _inputs[sourceString] = inputConnectionRecord; Console.WriteLine("Adding input:{0}", sourceString); } @@ -3581,7 +3657,7 @@ private async Task FromDataStreamAsync(Stream readFromStream, Console.WriteLine("restoring input:{0}", sourceString); } inputConnectionRecord.DataConnectionStream = (NetworkStream)readFromStream; - await SendReplayMessageAsync(readFromStream, inputConnectionRecord.LastProcessedID + 1, inputConnectionRecord.LastProcessedReplayableID + 1, ct); + await SendReplayMessageAsync(readFromStream, inputConnectionRecord, ct); // Create new input task for monitoring new input Task inputTask; inputTask = InputDataListenerAsync(inputConnectionRecord, sourceString, ct); @@ -3601,6 +3677,7 @@ private async Task FromControlStreamAsync(Stream readFromStream, { // Create input record and add it to the dictionary inputConnectionRecord = new InputConnectionRecord(); + inputConnectionRecord.ShardID = ParseServiceName(sourceString).Item2; _inputs[sourceString] = inputConnectionRecord; Console.WriteLine("Adding input:{0}", sourceString); } diff --git a/Ambrosia/Ambrosia/Serializer.cs b/Ambrosia/Ambrosia/Serializer.cs new file mode 100644 index 00000000..80b6d5ff --- /dev/null +++ b/Ambrosia/Ambrosia/Serializer.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Ambrosia +{ + public class Serializer + { + private const int messageTypeSize = 1; + + public static void SerializeReplayMessage(Stream stream, + byte messageType, + long lastProcessedID, + long lastProcessedReplayableID, + ConcurrentDictionary>> ancestorsToIDs) + { + var dictCount = ancestorsToIDs.Count; + var messageSize = messageTypeSize + StreamCommunicator.LongSize(lastProcessedID) + + StreamCommunicator.LongSize(lastProcessedReplayableID) + + StreamCommunicator.LongSize(dictCount); + + foreach (var peerID in ancestorsToIDs.Keys) + { + messageSize += StreamCommunicator.LongSize(peerID); + messageSize += StreamCommunicator.LongSize(ancestorsToIDs[peerID].Count); + foreach (var shardID in ancestorsToIDs[peerID].Keys) + { + messageSize += StreamCommunicator.LongSize(shardID); + messageSize += StreamCommunicator.LongSize(ancestorsToIDs[peerID][shardID].Item1); + messageSize += StreamCommunicator.LongSize(ancestorsToIDs[peerID][shardID].Item2); + } + } + + // Write message size + stream.WriteInt(messageSize); + // Write message type + stream.WriteByte(messageType); + // Write the output filter seqNo for the other side + stream.WriteLong(lastProcessedID); + stream.WriteLong(lastProcessedReplayableID); + + // For the sharded case, send replay values for other shards + stream.WriteInt(dictCount); + foreach (var peerID in ancestorsToIDs.Keys) + { + stream.WriteLong(peerID); + stream.WriteInt(ancestorsToIDs[peerID].Count); + foreach (var shardID in ancestorsToIDs[peerID].Keys) + { + stream.WriteLong(shardID); + stream.WriteLong(ancestorsToIDs[peerID][shardID].Item1); + stream.WriteLong(ancestorsToIDs[peerID][shardID].Item2); + } + } + } + + public static async Task>>>> DeserializeReplayMessageAsync(Stream stream, CancellationToken ct) + { + var inputFlexBuffer = new FlexReadBuffer(); + await FlexReadBuffer.DeserializeAsync(stream, inputFlexBuffer, ct); + var sizeBytes = inputFlexBuffer.LengthLength; + // Get the seqNo of the replay/filter point + var offset = messageTypeSize + sizeBytes; + var lastProcessedID = StreamCommunicator.ReadBufferedLong(inputFlexBuffer.Buffer, offset); + offset += StreamCommunicator.LongSize(lastProcessedID); + var lastProcessedReplayableID = StreamCommunicator.ReadBufferedLong(inputFlexBuffer.Buffer, offset); + offset += StreamCommunicator.LongSize(lastProcessedReplayableID); + var numPeers = StreamCommunicator.ReadBufferedInt(inputFlexBuffer.Buffer, offset); + offset += StreamCommunicator.IntSize(numPeers); + var ancestorsToIDs = new ConcurrentDictionary>>(); + for (int i = 0; i < numPeers; i++) + { + long peerID = StreamCommunicator.ReadBufferedLong(inputFlexBuffer.Buffer, offset); + offset += StreamCommunicator.LongSize(peerID); + int numShards = StreamCommunicator.ReadBufferedInt(inputFlexBuffer.Buffer, offset); + offset += StreamCommunicator.IntSize(numShards); + ancestorsToIDs[peerID] = new ConcurrentDictionary>(); + for (int j = 0; j < numShards; j++) + { + long shardID = StreamCommunicator.ReadBufferedLong(inputFlexBuffer.Buffer, offset); + offset += StreamCommunicator.LongSize(shardID); + long shardLastProcessedID = StreamCommunicator.ReadBufferedLong(inputFlexBuffer.Buffer, offset); + offset += StreamCommunicator.LongSize(shardLastProcessedID); + long shardLastProcessedReplayableID = StreamCommunicator.ReadBufferedLong(inputFlexBuffer.Buffer, offset); + offset += StreamCommunicator.LongSize(shardLastProcessedReplayableID); + ancestorsToIDs[peerID][shardID] = new Tuple(shardLastProcessedID, shardLastProcessedReplayableID); + } + } + inputFlexBuffer.ResetBuffer(); + return Tuple.Create(lastProcessedID, lastProcessedReplayableID, ancestorsToIDs); + } + } +} diff --git a/Ambrosia/AmbrosiaTests/AmbrosiaTests.csproj b/Ambrosia/AmbrosiaTests/AmbrosiaTests.csproj new file mode 100644 index 00000000..69016b40 --- /dev/null +++ b/Ambrosia/AmbrosiaTests/AmbrosiaTests.csproj @@ -0,0 +1,27 @@ + + + + netcoreapp2.1 + + false + + + + x64 + + + + x64 + + + + + + + + + + + + + diff --git a/Ambrosia/AmbrosiaTests/SerializerTests.cs b/Ambrosia/AmbrosiaTests/SerializerTests.cs new file mode 100644 index 00000000..8569d6a6 --- /dev/null +++ b/Ambrosia/AmbrosiaTests/SerializerTests.cs @@ -0,0 +1,82 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Ambrosia; +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Threading; + +namespace AmbrosiaTests +{ + [TestClass] + public class SerializerTests + { + [TestMethod] + public void TestNonShardReplayMessage() + { + long expectedLastProcessedID = 14; + long expectedLastProcessedReplayableID = 9; + var ancestorsToIDs = new ConcurrentDictionary>>(); + + using (var stream = new MemoryStream()) + { + Serializer.SerializeReplayMessage( + stream, + AmbrosiaRuntime.replayFromByte, + expectedLastProcessedID, + expectedLastProcessedReplayableID, + ancestorsToIDs + ); + stream.Flush(); + stream.Position = 0; + var result = Serializer.DeserializeReplayMessageAsync(stream, new CancellationToken()).GetAwaiter().GetResult(); + Assert.AreEqual(expectedLastProcessedID, result.Item1); + Assert.AreEqual(expectedLastProcessedReplayableID, result.Item2); + var actualShardToLastID = result.Item3; + Assert.AreEqual(actualShardToLastID.Count, 0); + } + } + + [TestMethod] + public void TestShardReplayMessage() + { + long expectedLastProcessedID = 14; + long expectedLastProcessedReplayableID = 9; + var ancestorsToIDs = new ConcurrentDictionary>>(); + ancestorsToIDs[2] = new ConcurrentDictionary>(); + ancestorsToIDs[2][1] = new Tuple(11, 7); + ancestorsToIDs[2][5] = new Tuple(9, 4); + ancestorsToIDs[4] = new ConcurrentDictionary>(); + ancestorsToIDs[4][3] = new Tuple(8, 6); + ancestorsToIDs[4][7] = new Tuple(10, 5); + + using (var stream = new MemoryStream()) + { + Serializer.SerializeReplayMessage( + stream, + AmbrosiaRuntime.replayFromByte, + expectedLastProcessedID, + expectedLastProcessedReplayableID, + ancestorsToIDs + ); + stream.Flush(); + stream.Position = 0; + var result = Serializer.DeserializeReplayMessageAsync(stream, new CancellationToken()).GetAwaiter().GetResult(); + Assert.AreEqual(expectedLastProcessedID, result.Item1); + Assert.AreEqual(expectedLastProcessedReplayableID, result.Item2); + var actualShardToLastID = result.Item3; + Assert.AreEqual(actualShardToLastID.Count, ancestorsToIDs.Count); + + foreach (var peerID in ancestorsToIDs.Keys) + { + Assert.IsTrue(actualShardToLastID.ContainsKey(peerID)); + Assert.AreEqual(actualShardToLastID[peerID].Count, ancestorsToIDs[peerID].Count); + foreach (var shardID in ancestorsToIDs[peerID].Keys) + { + Assert.IsTrue(actualShardToLastID[peerID].ContainsKey(shardID)); + Assert.AreEqual(actualShardToLastID[peerID][shardID], ancestorsToIDs[peerID][shardID]); + } + } + } + } + } +} From 40108f66a0ff24aad2f4983e1cd3d27304b2b4ec Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Mon, 10 Jun 2019 11:19:24 -0700 Subject: [PATCH 09/22] Basic setup for initializing sharded machines In the sharded case, we have to recover from multiple machines. We initialize the machine with a list of shards to recover from as well as a map indicating which machines each key belongs to. This also starts the merge process for the parent input records. This commit may be easier to view with -w. --- Ambrosia/Ambrosia/Program.cs | 304 +++++++++++++++++++++++++++++++++-- 1 file changed, 288 insertions(+), 16 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 5e1a4ef5..6af88b30 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -1091,6 +1091,8 @@ public class AmbrosiaRuntimeParams public long currentVersion; public long upgradeToVersion; public long shardID; + public long[] oldShards; + public long[] newShards; } public static class AmbrosiaRuntimeParms @@ -1976,6 +1978,7 @@ internal class MachineState { public MachineState(long shardID) { + Recovered = false; ShardID = shardID; } public LogWriter CheckpointWriter { get; set; } @@ -1985,6 +1988,7 @@ public MachineState(long shardID) public long LastLogFile { get; set; } public AARole MyRole { get; set; } public ConcurrentDictionary Outputs { get; set; } + public bool Recovered { get; set; } public long ShardID { get; set; } } @@ -2082,6 +2086,10 @@ public async Task FromStreamAsync(Stream stream, string otherProcess, string oth ConcurrentDictionary _inputs; ConcurrentDictionary _outputs; + // Input / output state of parent shards. This is needed for recovery. + + ConcurrentDictionary _parentStates = new ConcurrentDictionary(); + ConcurrentDictionary _ancestors; internal int _localServiceReceiveFromPort; // specifiable on the command line internal int _localServiceSendToPort; // specifiable on the command line internal string _serviceName; // specifiable on the command line @@ -2096,6 +2104,8 @@ public async Task FromStreamAsync(Stream stream, string otherProcess, string oth internal bool _createService; long _shardID; Func _shardLocator; + long[] _oldShards; + long[] _newShards; bool _runningRepro; long _currentVersion; long _upgradeToVersion; @@ -2145,7 +2155,7 @@ public async Task FromStreamAsync(Stream stream, string otherProcess, string oth // true when this service is in an active/active configuration. False if set to single node bool _activeActive; - internal enum AARole { Primary, Secondary, Checkpointer }; + internal enum AARole { Primary, Secondary, Checkpointer, ReshardSecondary }; AARole _myRole; // Log size at which we start a new log file. This triggers a checkpoint, <= 0 if manual only checkpointing is done long _newLogTriggerSize; @@ -2156,8 +2166,9 @@ internal enum AARole { Primary, Secondary, Checkpointer }; // A handle to a file used for an upgrading secondary to bring down the primary and prevent primary promotion amongst secondaries. // As long as the write lock is held, no promotion can happen FileStream _killFileHandle = null; - - + // A handle to a file used to signal to other shards that this shard has finished upgrading. + // When all of these files exists, then all shards know re-sharding is complete. + FileStream _reshardFileHandle = null; const int UnexpectedError = 0; const int VersionMismatch = 1; @@ -2339,9 +2350,17 @@ private async Task RecoverOrStartAsync(long checkpointToLoad = -1, { Recovering = true; _restartWithRecovery = true; - MachineState state = new MachineState(_shardID); - await RecoverAsync(state, checkpointToLoad, testUpgrade); - UpdateAmbrosiaState(state); + if (_oldShards.Length > 0) + { + RecoverFromShards(checkpointToLoad, testUpgrade); + } else + { + MachineState state = new MachineState(_shardID); + state.MyRole = AARole.Secondary; + // TODO: Need to figure out where _outputs should have been set if we replay re-sharding. + await RecoverAsync(state, checkpointToLoad, testUpgrade); + UpdateAmbrosiaState(state); + } await PrepareToBecomePrimaryAsync(); Recovering = false; } @@ -2442,13 +2461,197 @@ private async Task PrepareToBecomePrimaryAsync() await MoveServiceToNextLogFileAsync(); } - if (_activeActive) + if (_activeActive || _shardID > 0) { // Start task to periodically check if someone's trying to upgrade (new Task(() => CheckForUpgradeAsync())).Start(); } } + private void RecoverFromShard(MachineState state, long shardID, long checkpointToLoad = -1, bool testUpgrade = false) + { + RecoverAsync(state, checkpointToLoad, testUpgrade).GetAwaiter(); + } + + private void AddAncestorInput(string key, long inputShardID, long peerID, long shardID, Tuple seq) + { + InputConnectionRecord inputConnectionRecord; + if (!_inputs.TryGetValue(key, out inputConnectionRecord)) + { + inputConnectionRecord = new InputConnectionRecord(); + inputConnectionRecord.ShardID = inputShardID; + _inputs[key] = inputConnectionRecord; + } + if (!inputConnectionRecord.AncestorsToIDs.ContainsKey(peerID)) { + inputConnectionRecord.AncestorsToIDs[peerID] = new ConcurrentDictionary>(); + } + inputConnectionRecord.AncestorsToIDs[peerID][shardID] = seq; + } + + private void AddAncestorInput(string sourceString) + { + var srcAncestors = _ancestors[sourceString]; + var dstAncestors = _ancestors[ServiceName()]; + var parts = ParseServiceName(sourceString); + var service = parts.Item1; + foreach (var srcID in srcAncestors) + { + var srcName = ServiceName(service, srcID); + InputConnectionRecord record; + if (!_inputs.TryGetValue(srcName, out record)) + { + record = new InputConnectionRecord(); + _inputs[srcName] = record; + } + // Add our records for the source ancestor inputs to the source Input. + AddAncestorInput(sourceString, parts.Item2, srcID, _shardID, new Tuple(record.LastProcessedID, record.LastProcessedReplayableID)); + // Merge the ancestors AncestorsToIDs records with the sourceString's records. + foreach (var srcID1 in record.AncestorsToIDs.Keys) + { + foreach (var destID1 in record.AncestorsToIDs[srcID1].Keys) + { + AddAncestorInput(sourceString, parts.Item2, srcID1, destID1, record.AncestorsToIDs[srcID1][destID1]); + } + } + } + } + + private void AddParentInputState(MachineState state) + { + foreach (var input in state.Inputs) + { + var record = input.Value; + // Add the parent's ancestor history + foreach (var peerID in record.AncestorsToIDs.Keys) + { + foreach (var shardID in record.AncestorsToIDs[peerID].Keys) + { + AddAncestorInput(input.Key, record.ShardID, peerID, shardID, record.AncestorsToIDs[peerID][shardID]); + } + } + // Add the parents history + AddAncestorInput(input.Key, record.ShardID, record.ShardID, state.ShardID, new Tuple(record.LastProcessedID, record.LastProcessedReplayableID)); + } + } + + private void AddParentStates(MachineState[] states) + { + foreach (var state in states) + { + AddParentInputState(state); + } + } + + private void FinishResharding() + { + WaitForOtherShards(); + LockReshardFile(); + if (_newShards.Last() == _shardID) + { + KillParentShards(_oldShards); + } + } + + private void EstablishInputConnections(ConcurrentDictionary states) + { + foreach (var state in states.Values) + { + foreach (var kv in state.Inputs) + { + var destination = kv.Key; + if (destination == "") + { + destination = ServiceName(); + } + + List results = new List(); + results.Add(Connect(ServiceName(), AmbrosiaDataOutputsName, destination, AmbrosiaDataInputsName)); + results.Add(Connect(ServiceName(), AmbrosiaControlOutputsName, destination, AmbrosiaControlInputsName)); + if (destination != ServiceName()) + { + results.Add(Connect(destination, AmbrosiaDataOutputsName, ServiceName(), AmbrosiaDataInputsName)); + results.Add(Connect(destination, AmbrosiaControlOutputsName, ServiceName(), AmbrosiaControlInputsName)); + } + + foreach (var result in results) + { + if (result != CRAErrorCode.Success) + { + Console.WriteLine("EstablishInputConnections: Error connecting to " + destination); + break; + } + } + } + } + } + + private void RecoverFromShards(long checkpointToLoad = -1, bool testUpgrade = false) + { + _myRole = AARole.ReshardSecondary; + var threads = new Thread[_oldShards.Length]; + InsertOrReplaceServiceInfoRecord(InfoTitle("CurrentVersion"), _currentVersion.ToString()); + + _inputs = new ConcurrentDictionary(); + _outputs = new ConcurrentDictionary(); + _committer = new Committer(_localServiceSendToStream, _persistLogs, this); + + for (int i = 0; i < _oldShards.Length; i++) + { + long shardID = _oldShards[i]; + MachineState state = new MachineState(shardID); + state.MyRole = AARole.ReshardSecondary; + _parentStates[shardID] = state; + if (i == 0) + { + var t = DetectBecomingPrimaryAsync(state); + } + threads[i] = new Thread(() => RecoverFromShard(state, shardID, checkpointToLoad, testUpgrade)) { IsBackground = true }; + threads[i].Start(); + } + + while (!_parentStates.Values.All(s => s.Recovered)) + { + Thread.Sleep(1000); + } + + AddParentStates(_parentStates.Values.ToArray()); + EstablishInputConnections(_parentStates); + + // Wait for replay for all shards to occur + for (int i = 0; i < threads.Length; i++) + { + threads[i].Join(); + } + } + + private void WaitForOtherShards() + { + int index = -1; + for (int i = 0; i < _newShards.Length; i++) + { + if (_newShards[i] == _shardID) + { + index = i; + } + } + Debug.Assert(index != -1); + if (index > 0) + { + if (!File.Exists(ReshardFileName(_newShards[index - 1]))) + { + throw new Exception("Peer shard not done syncing."); + } + } + } + + private void KillParentShards(long[] shardIds) + { + for (int i = 0; i < shardIds.Length; i++) + { + LockKillFile(shardIds[i]); + } + } + private async Task StartAsync() { // We are starting for the first time. This is the primary @@ -2467,7 +2670,7 @@ private async Task StartAsync() Connect(ServiceName(), AmbrosiaControlOutputsName, ServiceName(), AmbrosiaControlInputsName); await MoveServiceToNextLogFileAsync(true, true); InsertOrReplaceServiceInfoRecord(InfoTitle("CurrentVersion"), _currentVersion.ToString()); - if (_activeActive) + if (_activeActive || _shardID > 0) { // Start task to periodically check if someone's trying to upgrade (new Task(() => CheckForUpgradeAsync())).Start(); @@ -2595,6 +2798,11 @@ private string KillFileName(long shardID = -1) return LogFileNameBase(-1, shardID) + "killFile"; } + private string ReshardFileName(long shardID = -1) + { + return LogFileNameBase(-1, shardID) + "reshardFile"; + } + private LogWriter CreateNextOldVerLogFile() { if (LogWriter.FileExists(LogFileName(_lastLogFile + 1, _currentVersion))) @@ -2620,6 +2828,11 @@ private void LockKillFile(long shardID = -1) _killFileHandle = LockFile(KillFileName(shardID)); } + private void LockReshardFile() + { + _reshardFileHandle = LockFile(ReshardFileName()); + } + private FileStream LockFile(string file) { return new FileStream(file, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read & ~FileShare.Inheritable); @@ -2891,7 +3104,7 @@ private async Task ReplayAsync(LogReader replayStream, MachineState state) catch { // Couldn't recover replay segment. Could be for a number of reasons. - if (!_activeActive || detectedEOL) + if (state.MyRole != AARole.ReshardSecondary && (!_activeActive || detectedEOL)) { // Leave replay and continue recovery. break; @@ -2951,7 +3164,7 @@ private async Task ReplayAsync(LogReader replayStream, MachineState state) continue; } // The remaining case is that we hit the end of log, but someone is still writing to this file. Wait and try to read again, or kill the primary if we are trying to upgrade in an active/active scenario - if (_upgrading && _activeActive && _killFileHandle == null) + if ((state.MyRole == AARole.ReshardSecondary || (_upgrading && _activeActive)) && _killFileHandle == null) { // We need to write and hold the lock on the kill file. Recovery will continue until the primary dies and we have // fully processed the log. @@ -2959,7 +3172,14 @@ private async Task ReplayAsync(LogReader replayStream, MachineState state) { try { - LockKillFile(); + if (state.MyRole == AARole.ReshardSecondary) + { + FinishResharding(); + } + else + { + LockKillFile(); + } break; } catch (Exception e) @@ -3038,6 +3258,7 @@ private async Task ReplayAsync(LogReader replayStream, MachineState state) // bump up the write ID in the committer in preparation for reading or writing the next page state.Committer._nextWriteID++; } + state.Recovered = true; } // Thread for listening to the local service @@ -3458,6 +3679,13 @@ private async Task ToDataStreamAsync(Stream writeToStream, // Get the seqNo of the replay/filter point var commitSeqNo = result.Item1; var commitSeqNoReplayable = result.Item2; + var shardToLastID = result.Item3; + var service = destString.Split('_')[0]; + + foreach (var kv in shardToLastID) + { + await SyncOutputConnectionAsync(_outputs, ServiceName(service, kv.Key), commitSeqNo, commitSeqNoReplayable); + } OutputConnectionRecord outputConnectionRecord = await SyncOutputConnectionAsync(_outputs, destString, commitSeqNo, commitSeqNoReplayable); @@ -3999,6 +4227,8 @@ public override void Initialize(object param) bool sharded = p.shardID > 0; _shardID = p.shardID; _shardLocator = KeyHashToShard; + _oldShards = p.oldShards; + _newShards = p.newShards; Initialize( p.serviceReceiveFromPort, @@ -4025,6 +4255,7 @@ public void Initialize(object param, long[] oldShards, long shardID, Func _replicaNumber = int.Parse(r) }, }.AddMany(registerInstanceOptionSet); + var addShardOptionSet = new OptionSet + { + {"os|oldShards=", "Comma separated list of shards to recover from [REQUIRED].", os => _oldShards = os }, + {"ns|newShards=", "Comma separated list of new shards being created [REQUIRED].", ns => _newShards = ns } + }.AddMany(registerInstanceOptionSet); + var debugInstanceOptionSet = basicOptions.AddMany(new OptionSet { { "c|checkpoint=", "The checkpoint # to load.", c => _checkpointToLoad = long.Parse(c) }, @@ -4322,6 +4584,7 @@ private static OptionSet ParseOptions(string[] args, out bool shouldShowHelp) registerInstanceOptionSet = registerInstanceOptionSet.AddMany(helpOption); addReplicaOptionSet = addReplicaOptionSet.AddMany(helpOption); + addShardOptionSet = addShardOptionSet.AddMany(helpOption); debugInstanceOptionSet = debugInstanceOptionSet.AddMany(helpOption); @@ -4329,6 +4592,7 @@ private static OptionSet ParseOptions(string[] args, out bool shouldShowHelp) { { LocalAmbrosiaRuntimeModes.RegisterInstance, registerInstanceOptionSet}, { LocalAmbrosiaRuntimeModes.AddReplica, addReplicaOptionSet}, + { LocalAmbrosiaRuntimeModes.AddShard, addShardOptionSet }, { LocalAmbrosiaRuntimeModes.DebugInstance, debugInstanceOptionSet}, }; @@ -4360,6 +4624,7 @@ private static OptionSet ParseOptions(string[] args, out bool shouldShowHelp) public enum LocalAmbrosiaRuntimeModes { AddReplica, + AddShard, RegisterInstance, DebugInstance, } @@ -4395,6 +4660,13 @@ private static void ValidateOptions(OptionSet options, bool shouldShowHelp) errorMessage += "Replica number is required.\n"; } } + if (_runtimeMode == LocalAmbrosiaRuntimeModes.AddShard) + { + if (_shardID == -1) + { + errorMessage += "Shard ID is required.\n"; + } + } // handles the case when an upgradeversion is not specified if (_upgradeVersion == -1) From d6c71cd7b8f13e7713d26c2918e3951abc834a99 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Mon, 1 Jul 2019 09:33:05 -0700 Subject: [PATCH 10/22] Track ancestor information Create a dictionary to keep track of the ancestors associated with each shard ID. This information should not change, so we store it with service information. The ancestor information will help determine which input / output information needs to be shared with peers. --- Ambrosia/Ambrosia/Program.cs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 6af88b30..38c0a474 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -2087,7 +2087,6 @@ public async Task FromStreamAsync(Stream stream, string otherProcess, string oth ConcurrentDictionary _inputs; ConcurrentDictionary _outputs; // Input / output state of parent shards. This is needed for recovery. - ConcurrentDictionary _parentStates = new ConcurrentDictionary(); ConcurrentDictionary _ancestors; internal int _localServiceReceiveFromPort; // specifiable on the command line @@ -2345,6 +2344,12 @@ private async Task RecoverOrStartAsync(long checkpointToLoad = -1, RuntimeChecksOnProcessStart(); } + if (_sharded) + { + _ancestors = new ConcurrentDictionary(); + _ancestors[ServiceName()] = _oldShards; + } + // Determine if we are recovering if (!_createService) { @@ -2376,6 +2381,12 @@ private async Task RecoverAsync(MachineState state, long checkpointToLoad = -1, { // We are recovering - find the last committed checkpoint state.LastCommittedCheckpoint = long.Parse(RetrieveServiceInfo(InfoTitle("LastCommittedCheckpoint", state.ShardID))); + if (_sharded && state.MyRole != AARole.ReshardSecondary) + { + // Load our ancestor information + string ancestors = RetrieveServiceInfo(InfoTitle("Ancestors", state.ShardID)); + _ancestors[ServiceName()] = ancestors.Split(',').Select(n => long.Parse(n)).ToArray(); + } } else { @@ -2590,6 +2601,7 @@ private void RecoverFromShards(long checkpointToLoad = -1, bool testUpgrade = fa _myRole = AARole.ReshardSecondary; var threads = new Thread[_oldShards.Length]; InsertOrReplaceServiceInfoRecord(InfoTitle("CurrentVersion"), _currentVersion.ToString()); + InsertOrReplaceServiceInfoRecord(InfoTitle("Ancestors"), string.Join(",", _ancestors[ServiceName()])); _inputs = new ConcurrentDictionary(); _outputs = new ConcurrentDictionary(); @@ -2670,6 +2682,10 @@ private async Task StartAsync() Connect(ServiceName(), AmbrosiaControlOutputsName, ServiceName(), AmbrosiaControlInputsName); await MoveServiceToNextLogFileAsync(true, true); InsertOrReplaceServiceInfoRecord(InfoTitle("CurrentVersion"), _currentVersion.ToString()); + if (_sharded) + { + InsertOrReplaceServiceInfoRecord(InfoTitle("Ancestors"), string.Join(",", _ancestors[ServiceName()])); + } if (_activeActive || _shardID > 0) { // Start task to periodically check if someone's trying to upgrade From f8375b4c139d038c9f2c2c20e090e54113b2c927 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Mon, 1 Jul 2019 10:12:44 -0700 Subject: [PATCH 11/22] Send ancestor information to peers When a connection is established, before the peers send a replay message, we send our ancestor list, so the peer knows which shard input / output data to add to the replay message. --- Ambrosia/Ambrosia/Program.cs | 13 ++++++++ Ambrosia/Ambrosia/Serializer.cs | 40 +++++++++++++++++++++++ Ambrosia/AmbrosiaTests/SerializerTests.cs | 18 ++++++++++ 3 files changed, 71 insertions(+) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 38c0a474..2bbbe263 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -2129,6 +2129,7 @@ public async Task FromStreamAsync(Stream stream, string otherProcess, string oth public const byte CountReplayableRPCBatchByte = 13; public const byte trimToByte = 14; public const byte becomingPrimaryByte = 15; + public const byte ancestorByte = 16; CRAClientLibrary _coral; @@ -3690,6 +3691,10 @@ private async Task ToDataStreamAsync(Stream writeToStream, string destString, CancellationToken ct) { + if (_sharded) + { + Serializer.SerializeAncestorMessage(writeToStream, ancestorByte, _ancestors[ServiceName()]); + } // Process replay message var result = await Serializer.DeserializeReplayMessageAsync(writeToStream, ct); // Get the seqNo of the replay/filter point @@ -3901,6 +3906,14 @@ private async Task FromDataStreamAsync(Stream readFromStream, Console.WriteLine("restoring input:{0}", sourceString); } inputConnectionRecord.DataConnectionStream = (NetworkStream)readFromStream; + if (_sharded) + { + // Process ancestor list + _ancestors[sourceString] = await Serializer.DeserializeAncestorMessageAsync(readFromStream, ct); + // The last destination may be an ancestor shard, in which case we need to reshuffle. + _lastShuffleDestSize = 0; + Console.WriteLine("Ancestors of shard " + sourceString + " are " + string.Join(",", _ancestors[sourceString])); + } await SendReplayMessageAsync(readFromStream, inputConnectionRecord, ct); // Create new input task for monitoring new input Task inputTask; diff --git a/Ambrosia/Ambrosia/Serializer.cs b/Ambrosia/Ambrosia/Serializer.cs index 80b6d5ff..f7740c8d 100644 --- a/Ambrosia/Ambrosia/Serializer.cs +++ b/Ambrosia/Ambrosia/Serializer.cs @@ -10,6 +10,46 @@ public class Serializer { private const int messageTypeSize = 1; + public static void SerializeAncestorMessage(Stream stream, byte messageType, long[] ancestors) + { + var numAncestors = ancestors.Length; + var messageSize = messageTypeSize + StreamCommunicator.LongSize(numAncestors); + foreach (var ancestor in ancestors) + { + messageSize += StreamCommunicator.LongSize(ancestor); + } + + // Write message size + stream.WriteInt(messageSize); + // Write message type + stream.WriteByte(messageType); + // Write number of ancestors + stream.WriteInt(numAncestors); + // Write ancestors + foreach(var ancestor in ancestors) + { + stream.WriteLong(ancestor); + } + } + + public static async Task DeserializeAncestorMessageAsync(Stream stream, CancellationToken ct) + { + var inputFlexBuffer = new FlexReadBuffer(); + await FlexReadBuffer.DeserializeAsync(stream, inputFlexBuffer, ct); + var sizeBytes = inputFlexBuffer.LengthLength; + // Get the seqNo of the replay/filter point + var offset = messageTypeSize + sizeBytes; + var numAncestors = StreamCommunicator.ReadBufferedInt(inputFlexBuffer.Buffer, offset); + offset += StreamCommunicator.IntSize(numAncestors); + var ancestors = new long[numAncestors]; + for (int i = 0; i < numAncestors; i++) + { + ancestors[i] = StreamCommunicator.ReadBufferedLong(inputFlexBuffer.Buffer, offset); + offset += StreamCommunicator.LongSize(ancestors[i]); + } + return ancestors; + } + public static void SerializeReplayMessage(Stream stream, byte messageType, long lastProcessedID, diff --git a/Ambrosia/AmbrosiaTests/SerializerTests.cs b/Ambrosia/AmbrosiaTests/SerializerTests.cs index 8569d6a6..3e5c7eb4 100644 --- a/Ambrosia/AmbrosiaTests/SerializerTests.cs +++ b/Ambrosia/AmbrosiaTests/SerializerTests.cs @@ -10,6 +10,24 @@ namespace AmbrosiaTests [TestClass] public class SerializerTests { + [TestMethod] + public void TestAncestorMessage() + { + long[] expectedAncestors = new long[3] { 3, 17, 39 }; + using (var stream = new MemoryStream()) + { + Serializer.SerializeAncestorMessage( + stream, + AmbrosiaRuntime.ancestorByte, + expectedAncestors + ); + stream.Flush(); + stream.Position = 0; + var actualAncestors = Serializer.DeserializeAncestorMessageAsync(stream, new CancellationToken()).GetAwaiter().GetResult(); + CollectionAssert.AreEqual(expectedAncestors, actualAncestors); + } + } + [TestMethod] public void TestNonShardReplayMessage() { From f4d10bc9305ff67908f01b7e4b2b2b8a4ceebd02 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Tue, 2 Jul 2019 09:23:35 -0700 Subject: [PATCH 12/22] Trim output based on replay ancestor data We clear the ancestor data as we know the shard received the data and we no longer need to track this data. --- Ambrosia/Ambrosia/Program.cs | 70 +++++++++++++++++++++++---------- Ambrosia/Ambrosia/Serializer.cs | 21 +++++++++- 2 files changed, 69 insertions(+), 22 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 2bbbe263..e147d890 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -2130,6 +2130,7 @@ public async Task FromStreamAsync(Stream stream, string otherProcess, string oth public const byte trimToByte = 14; public const byte becomingPrimaryByte = 15; public const byte ancestorByte = 16; + public const byte shardTrimByte = 17; CRAClientLibrary _coral; @@ -3687,6 +3688,19 @@ private async Task SyncOutputConnectionAsync(ConcurrentD return outputConnectionRecord; } + private void SyncOutputs(string destString, ConcurrentDictionary>> ancestorsToIDs) + { + var destService = ParseServiceName(destString).Item1; + foreach (var srcID in ancestorsToIDs.Keys) + { + foreach (var destID in ancestorsToIDs[srcID].Keys) + { + var destName = ServiceName(destService, destID); + TrimOutput(destName, ancestorsToIDs[srcID][destID].Item1, ancestorsToIDs[srcID][destID].Item1); + } + } + } + private async Task ToDataStreamAsync(Stream writeToStream, string destString, CancellationToken ct) @@ -3700,15 +3714,18 @@ private async Task ToDataStreamAsync(Stream writeToStream, // Get the seqNo of the replay/filter point var commitSeqNo = result.Item1; var commitSeqNoReplayable = result.Item2; - var shardToLastID = result.Item3; - var service = destString.Split('_')[0]; - foreach (var kv in shardToLastID) + if (_sharded) { - await SyncOutputConnectionAsync(_outputs, ServiceName(service, kv.Key), commitSeqNo, commitSeqNoReplayable); + var ancestorsToIDs = result.Item3; + SyncOutputs(destString, ancestorsToIDs); } OutputConnectionRecord outputConnectionRecord = await SyncOutputConnectionAsync(_outputs, destString, commitSeqNo, commitSeqNoReplayable); + if (_sharded) + { + Serializer.SerializeShardTrimMessage(writeToStream, shardTrimByte); + } try { @@ -3910,11 +3927,18 @@ private async Task FromDataStreamAsync(Stream readFromStream, { // Process ancestor list _ancestors[sourceString] = await Serializer.DeserializeAncestorMessageAsync(readFromStream, ct); + AddAncestorInput(sourceString); // The last destination may be an ancestor shard, in which case we need to reshuffle. _lastShuffleDestSize = 0; Console.WriteLine("Ancestors of shard " + sourceString + " are " + string.Join(",", _ancestors[sourceString])); } await SendReplayMessageAsync(readFromStream, inputConnectionRecord, ct); + if (_sharded) + { + await Serializer.DeserializeShardTrimMessageAsync(readFromStream, ct); + // Clear ancestor history + _inputs[sourceString].AncestorsToIDs.Clear(); + } // Create new input task for monitoring new input Task inputTask; inputTask = InputDataListenerAsync(inputConnectionRecord, sourceString, ct); @@ -4150,6 +4174,27 @@ private void CleanupOldCheckpoint() } } + public void TrimOutput(string key, long lastProcessedID, long lastProcessedReplayableID) + { + Console.WriteLine("Trimming {0} up to ({1}, {2})", key, lastProcessedID, lastProcessedReplayableID); + OutputConnectionRecord outputConnectionRecord; + if (!_outputs.TryGetValue(key, out outputConnectionRecord)) + { + outputConnectionRecord = new OutputConnectionRecord(this); + _outputs[key] = outputConnectionRecord; + } + // Must lock to atomically update due to race with ToControlStreamAsync + lock (outputConnectionRecord._remoteTrimLock) + { + outputConnectionRecord.RemoteTrim = Math.Max(lastProcessedID, outputConnectionRecord.RemoteTrim); + outputConnectionRecord.RemoteTrimReplayable = Math.Max(lastProcessedReplayableID, outputConnectionRecord.RemoteTrimReplayable); + } + if (outputConnectionRecord.ControlWorkQ.IsEmpty) + { + outputConnectionRecord.ControlWorkQ.Enqueue(-2); + } + } + // This method takes a checkpoint and bumps the counter. It DOES NOT quiesce anything public async Task CheckpointAsync() { @@ -4188,22 +4233,7 @@ public async Task CheckpointAsync() // successfully written foreach (var kv in _inputs) { - OutputConnectionRecord outputConnectionRecord; - if (!_outputs.TryGetValue(kv.Key, out outputConnectionRecord)) - { - outputConnectionRecord = new OutputConnectionRecord(this); - _outputs[kv.Key] = outputConnectionRecord; - } - // Must lock to atomically update due to race with ToControlStreamAsync - lock (outputConnectionRecord._remoteTrimLock) - { - outputConnectionRecord.RemoteTrim = Math.Max(kv.Value.LastProcessedID, outputConnectionRecord.RemoteTrim); - outputConnectionRecord.RemoteTrimReplayable = Math.Max(kv.Value.LastProcessedReplayableID, outputConnectionRecord.RemoteTrimReplayable); - } - if (outputConnectionRecord.ControlWorkQ.IsEmpty) - { - outputConnectionRecord.ControlWorkQ.Enqueue(-2); - } + TrimOutput(kv.Key, kv.Value.LastProcessedID, kv.Value.LastProcessedReplayableID); } if (oldCheckpointWriter != null) diff --git a/Ambrosia/Ambrosia/Serializer.cs b/Ambrosia/Ambrosia/Serializer.cs index f7740c8d..8f8127b4 100644 --- a/Ambrosia/Ambrosia/Serializer.cs +++ b/Ambrosia/Ambrosia/Serializer.cs @@ -68,8 +68,8 @@ public static void SerializeReplayMessage(Stream stream, foreach (var shardID in ancestorsToIDs[peerID].Keys) { messageSize += StreamCommunicator.LongSize(shardID); - messageSize += StreamCommunicator.LongSize(ancestorsToIDs[peerID][shardID].Item1); - messageSize += StreamCommunicator.LongSize(ancestorsToIDs[peerID][shardID].Item2); + messageSize += StreamCommunicator.LongSize(ancestorsToIDs[peerID][shardID].Item1 + 1); + messageSize += StreamCommunicator.LongSize(ancestorsToIDs[peerID][shardID].Item2 + 1); } } @@ -131,5 +131,22 @@ public static async Task Date: Wed, 3 Jul 2019 11:02:54 -0700 Subject: [PATCH 13/22] Add shard output record recovery logic We need to preserve a global ordering of output records during shard recovery. If we fail before recovery completes, it's possible that the order of execution will change. To handle this, we record a global ordering of outputs. We also handle merging parent output state in this commit. --- Ambrosia/Ambrosia/Program.cs | 130 +++++++++++++++++++++++++++++------ 1 file changed, 110 insertions(+), 20 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index e147d890..e2b163a0 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -208,6 +208,7 @@ internal static void AmbrosiaSerialize(this ConcurrentDictionary AmbrosiaDes newRecord.TrimTo = readFromStream.ReadLongFixed(); newRecord.ReplayableTrimTo = readFromStream.ReadLongFixed(); newRecord.BufferedOutput = EventBuffer.Deserialize(readFromStream, thisAmbrosia, newRecord); + newRecord.OutputTracker.AmbrosiaDeserialize(readFromStream); _retVal.TryAdd(myString, newRecord); } return _retVal; } + + internal static void AmbrosiaSerialize(this ConcurrentQueue queue, LogWriter writeToStream) + { + writeToStream.WriteIntFixed(queue.Count); + foreach (var tracker in queue) + { + writeToStream.WriteLongFixed(tracker.GlobalSeqID); + writeToStream.WriteLongFixed(tracker.OutputSeqID); + } + } + + internal static ConcurrentQueue AmbrosiaDeserialize(this ConcurrentQueue queue, LogReader readFromStream) + { + var _retVal = new ConcurrentQueue(); + var queueCount = readFromStream.ReadIntFixed(); + for (int i = 0; i < queueCount; i++) + { + var globalSeqID = readFromStream.ReadLongFixed(); + var outputSeqID = readFromStream.ReadLongFixed(); + var tracker = new OutputRecordTracker(globalSeqID, outputSeqID); + _retVal.Enqueue(tracker); + } + return _retVal; + } } // Note about this class: contention becomes significant when MaxBufferPages > ~50. This could be reduced by having page level locking. @@ -661,6 +687,11 @@ internal async Task SendAsync(Stream outputStream, return placeToStart; } + internal IEnumerator GetEnumerator() + { + return _bufferQ.GetEnumerator(); + } + internal async Task ReplayFromAsync(Stream outputStream, long firstSeqNo, bool reconnecting) @@ -1021,6 +1052,19 @@ public InputConnectionRecord() } } + internal class OutputRecordTracker + { + public long GlobalSeqID { get; set; } + public long OutputSeqID { get; set; } + public bool Received { get; set; } + public OutputRecordTracker(long globalSeqID, long outputSeqID) + { + GlobalSeqID = globalSeqID; + OutputSeqID = outputSeqID; + Received = false; + } + } + internal class OutputConnectionRecord { // Set on reconnection. Established where to replay from or filter to @@ -1053,12 +1097,14 @@ internal class OutputConnectionRecord internal volatile bool ResettingConnection; internal object _trimLock = new object(); internal object _remoteTrimLock = new object(); + public ConcurrentQueue OutputTracker { get; set; } public OutputConnectionRecord(AmbrosiaRuntime inAmbrosia) { ReplayFrom = 0; DataWorkQ = new AsyncQueue(); ControlWorkQ = new AsyncQueue(); + OutputTracker = new ConcurrentQueue(); _sendsEnqueued = 0; TrimTo = -1; ReplayableTrimTo = -1; @@ -1073,6 +1119,40 @@ public OutputConnectionRecord(AmbrosiaRuntime inAmbrosia) WillResetConnection = inAmbrosia._createService; ConnectingAfterRestart = inAmbrosia._restartWithRecovery; } + + public void UpdateTracker() + { + try + { + var tracker = OutputTracker.First(); + var firstSeqID = tracker.OutputSeqID; + while (tracker.OutputSeqID < RemoteTrim) + { + tracker.Received = true; + OutputTracker.TryDequeue(out tracker); + tracker = OutputTracker.First(); + } + } + catch (InvalidOperationException) + { + // Queue is empty + } + } + + public void Trim(long lastProcessedID, long lastProcessedReplayableID) + { + // Must lock to atomically update due to race between ToControlStreamAsync, CheckpointAsync, and other functions + lock (_remoteTrimLock) + { + RemoteTrim = Math.Max(lastProcessedID, RemoteTrim); + RemoteTrimReplayable = Math.Max(lastProcessedReplayableID, RemoteTrimReplayable); + UpdateTracker(); + } + if (ControlWorkQ.IsEmpty) + { + ControlWorkQ.Enqueue(-2); + } + } } public class AmbrosiaRuntimeParams @@ -1309,16 +1389,7 @@ private void SendInputWatermarks(ConcurrentDictionary uncommit outputs[kv.Key] = outputConnectionRecord; Console.WriteLine("Adding output:{0}", kv.Key); } - // Must lock to atomically update due to race with ToControlStreamAsync - lock (outputConnectionRecord._remoteTrimLock) - { - outputConnectionRecord.RemoteTrim = Math.Max(kv.Value.First, outputConnectionRecord.RemoteTrim); - outputConnectionRecord.RemoteTrimReplayable = Math.Max(kv.Value.Second, outputConnectionRecord.RemoteTrimReplayable); - } - if (outputConnectionRecord.ControlWorkQ.IsEmpty) - { - outputConnectionRecord.ControlWorkQ.Enqueue(-2); - } + outputConnectionRecord.Trim(kv.Value.First, kv.Value.Second); } } } @@ -2084,6 +2155,9 @@ public async Task FromStreamAsync(Stream stream, string otherProcess, string oth } } + internal readonly object _globalOutputTrackerLock = new object(); + ConcurrentQueue _globalOutputTracker = new ConcurrentQueue(); + internal long _globalID = 0; ConcurrentDictionary _inputs; ConcurrentDictionary _outputs; // Input / output state of parent shards. This is needed for recovery. @@ -2552,6 +2626,7 @@ private void AddParentStates(MachineState[] states) foreach (var state in states) { AddParentInputState(state); + AddParentOutputState(state); } } @@ -2942,6 +3017,7 @@ private async Task MoveServiceToNextLogFileAsync(bool firstStart = fa // it's after replace and moving to the next log file. Note that this will also have the effect // of shaking loose the initialization message, ensuring liveliness. await _committer.TryCommitAsync(_outputs); + TrimGlobalOutputTracker(); return oldVerLogHandle; } @@ -4174,6 +4250,28 @@ private void CleanupOldCheckpoint() } } + private void TrimGlobalOutputTracker() + { + lock (_globalOutputTrackerLock) + { + try + { + var tracker = _globalOutputTracker.First(); + while (tracker.Received) + { + _globalOutputTracker.TryDequeue(out tracker); + tracker = _globalOutputTracker.First(); + + } + } + catch (InvalidOperationException) + { + // Queue is empty + } + + } + } + public void TrimOutput(string key, long lastProcessedID, long lastProcessedReplayableID) { Console.WriteLine("Trimming {0} up to ({1}, {2})", key, lastProcessedID, lastProcessedReplayableID); @@ -4183,16 +4281,8 @@ public void TrimOutput(string key, long lastProcessedID, long lastProcessedRepla outputConnectionRecord = new OutputConnectionRecord(this); _outputs[key] = outputConnectionRecord; } - // Must lock to atomically update due to race with ToControlStreamAsync - lock (outputConnectionRecord._remoteTrimLock) - { - outputConnectionRecord.RemoteTrim = Math.Max(lastProcessedID, outputConnectionRecord.RemoteTrim); - outputConnectionRecord.RemoteTrimReplayable = Math.Max(lastProcessedReplayableID, outputConnectionRecord.RemoteTrimReplayable); - } - if (outputConnectionRecord.ControlWorkQ.IsEmpty) - { - outputConnectionRecord.ControlWorkQ.Enqueue(-2); - } + outputConnectionRecord.Trim(lastProcessedID, lastProcessedReplayableID); + TrimGlobalOutputTracker(); } // This method takes a checkpoint and bumps the counter. It DOES NOT quiesce anything From 0df30af0c234a2760ba46b70ec797f367748c409 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Thu, 11 Jul 2019 14:31:38 -0700 Subject: [PATCH 14/22] Basic Shard Test This shard test tests basically the same thing as the normal basic end to end test, except the coordinators use the sharded logic. This is to ensure that the basic behavior for the sharded case is the same as the non-sharded case. --- AmbrosiaTest/AmbrosiaTest/AmbrosiaTest.csproj | 1 + .../Cmp/shardunitendtoendtest_AMB1.cmp | 1 + .../Cmp/shardunitendtoendtest_AMB2.cmp | 1 + .../Cmp/shardunitendtoendtest_ClientJob.cmp | 5 + ...shardunitendtoendtest_ClientJob_Verify.cmp | 5 + .../Cmp/shardunitendtoendtest_Server.cmp | 7 + .../shardunitendtoendtest_Server_Verify.cmp | 4 + AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs | 232 ++++++++++++++++++ AmbrosiaTest/AmbrosiaTest/Utilities.cs | 30 ++- 9 files changed, 283 insertions(+), 3 deletions(-) create mode 100644 AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_AMB1.cmp create mode 100644 AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_AMB2.cmp create mode 100644 AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_ClientJob.cmp create mode 100644 AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_ClientJob_Verify.cmp create mode 100644 AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_Server.cmp create mode 100644 AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_Server_Verify.cmp create mode 100644 AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs diff --git a/AmbrosiaTest/AmbrosiaTest/AmbrosiaTest.csproj b/AmbrosiaTest/AmbrosiaTest/AmbrosiaTest.csproj index 0b2bee08..c7a44d86 100644 --- a/AmbrosiaTest/AmbrosiaTest/AmbrosiaTest.csproj +++ b/AmbrosiaTest/AmbrosiaTest/AmbrosiaTest.csproj @@ -78,6 +78,7 @@ + diff --git a/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_AMB1.cmp b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_AMB1.cmp new file mode 100644 index 00000000..3797d308 --- /dev/null +++ b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_AMB1.cmp @@ -0,0 +1 @@ +The CRA instance appears to be down. Restart it and this vertex will be instantiated automatically diff --git a/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_AMB2.cmp b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_AMB2.cmp new file mode 100644 index 00000000..3797d308 --- /dev/null +++ b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_AMB2.cmp @@ -0,0 +1 @@ +The CRA instance appears to be down. Restart it and this vertex will be instantiated automatically diff --git a/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_ClientJob.cmp b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_ClientJob.cmp new file mode 100644 index 00000000..65203280 --- /dev/null +++ b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_ClientJob.cmp @@ -0,0 +1,5 @@ +Bytes per RPC Throughput (GB/sec) +*X* 1024 0.0046468462919506 +Service Received 1024 MB so far +Bytes received: 1073741824 +DONE diff --git a/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_ClientJob_Verify.cmp b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_ClientJob_Verify.cmp new file mode 100644 index 00000000..741adee4 --- /dev/null +++ b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_ClientJob_Verify.cmp @@ -0,0 +1,5 @@ +Bytes per RPC Throughput (GB/sec) +*X* 1024 0.0178702141848639 +Service Received 1024 MB so far +Bytes received: 1073741824 +DONE diff --git a/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_Server.cmp b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_Server.cmp new file mode 100644 index 00000000..49f4df51 --- /dev/null +++ b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_Server.cmp @@ -0,0 +1,7 @@ +*X* At checkpoint, received 0 messages +*X* Becoming a primary now +*X* Server in Entry Point +*X* At checkpoint, received 969264 messages +Received 1024 MB so far +Bytes received: 1073741824 +DONE diff --git a/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_Server_Verify.cmp b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_Server_Verify.cmp new file mode 100644 index 00000000..dcafc109 --- /dev/null +++ b/AmbrosiaTest/AmbrosiaTest/Cmp/shardunitendtoendtest_Server_Verify.cmp @@ -0,0 +1,4 @@ +*X* Server in Entry Point +Received 1024 MB so far +Bytes received: 1073741824 +DONE diff --git a/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs b/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs new file mode 100644 index 00000000..480f5d49 --- /dev/null +++ b/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs @@ -0,0 +1,232 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Configuration; +using System.Threading; + +namespace AmbrosiaTest +{ + [TestClass] + public class Shard_UnitTest + { + //************* Init Code ***************** + // NOTE: Need this bit of code at the top of every "[TestClass]" (per .cs test file) to get context \ details of the current test running + // NOTE: Make sure all names be "Azure Safe". No capital letters and no underscore. + [TestInitialize()] + public void Initialize() + { + Utilities MyUtils = new Utilities(); + MyUtils.TestInitialize(); + } + //************* Init Code ***************** + + [TestMethod] + public void Shard_UnitTest_BasicEndtoEnd_Test() + { + // Test that one shard per server and client works + string testName = "shardunitendtoendtest"; + string clientJobName = testName + "clientjob"; + string serverName = testName + "server"; + string ambrosiaLogDir = ConfigurationManager.AppSettings["AmbrosiaLogDirectory"] + "\\"; + string byteSize = "1073741824"; + + Utilities MyUtils = new Utilities(); + + // AMB1 - Job + string logOutputFileName_AMB1 = testName + "_AMB1.log"; + AMB_Settings AMB1 = new AMB_Settings + { + AMB_ServiceName = clientJobName, + AMB_PortAppReceives = "1000", + AMB_PortAMBSends = "1001", + AMB_ServiceLogPath = ambrosiaLogDir, + AMB_CreateService = "A", + AMB_PauseAtStart = "N", + AMB_PersistLogs = "Y", + AMB_NewLogTriggerSize = "1000", + AMB_ActiveActive = "N", + AMB_Version = "0", + AMB_ShardID = "1", + }; + MyUtils.CallAMB(AMB1, logOutputFileName_AMB1, AMB_ModeConsts.RegisterInstance); + + // AMB2 - Shard 1 + string logOutputFileName_AMB2 = testName + "_AMB2.log"; + AMB_Settings AMB2 = new AMB_Settings + { + AMB_ServiceName = serverName, + AMB_PortAppReceives = "2000", + AMB_PortAMBSends = "2001", + AMB_ServiceLogPath = ambrosiaLogDir, + AMB_CreateService = "A", + AMB_PauseAtStart = "N", + AMB_PersistLogs = "Y", + AMB_NewLogTriggerSize = "1000", + AMB_ActiveActive = "N", + AMB_Version = "0", + AMB_ShardID = "1", + }; + MyUtils.CallAMB(AMB2, logOutputFileName_AMB2, AMB_ModeConsts.RegisterInstance); + + // ImmCoord1 + string logOutputFileName_ImmCoord1 = testName + "_ImmCoord1.log"; + int ImmCoordProcessID1 = MyUtils.StartImmCoord(clientJobName, 1500, logOutputFileName_ImmCoord1, shardID: 1); + + // ImmCoord2 + string logOutputFileName_ImmCoord2 = testName + "_ImmCoord2.log"; + int ImmCoordProcessID2 = MyUtils.StartImmCoord(serverName, 2500, logOutputFileName_ImmCoord2, shardID: 1); + + // Client + string logOutputFileName_ClientJob = testName + "_ClientJob.log"; + int clientJobProcessID = MyUtils.StartPerfClientJob("1001", "1000", clientJobName, serverName, "1024", "1", logOutputFileName_ClientJob); + + // Give it a few seconds to start + Thread.Sleep(2000); + + // Server Call + string logOutputFileName_Server = testName + "_Server.log"; + int serverProcessID = MyUtils.StartPerfServer("2001", "2000", clientJobName, serverName, logOutputFileName_Server, 1, false); + + // Delay until client is done - also check Server just to make sure + bool pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob, byteSize, 5, false, testName, true); // Number of bytes processed + pass = MyUtils.WaitForProcessToFinish(logOutputFileName_Server, byteSize, 5, false, testName, true); + + // Stop things to file is freed up and can be opened in verify + MyUtils.KillProcess(clientJobProcessID); + MyUtils.KillProcess(serverProcessID); + MyUtils.KillProcess(ImmCoordProcessID1); + MyUtils.KillProcess(ImmCoordProcessID2); + + // Verify AMB + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_AMB1); + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_AMB2); + + // Verify Client + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_ClientJob); + + // Verify Server + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_Server); + + // Verify integrity of Ambrosia logs by replaying + MyUtils.VerifyAmbrosiaLogFile(testName, Convert.ToInt64(byteSize), true, true, AMB1.AMB_Version, shardID: 1); + } +/* + [TestMethod] + public void Shard_UnitTest_MultiShardEndToEnd_Test() + { + // Test that multi-shards per server and client behaves the same as the single shard case + string testName = "shardunitendtoendtest"; + string clientJobName = testName + "clientjob"; + string serverName = testName + "server"; + string ambrosiaLogDir = ConfigurationManager.AppSettings["AmbrosiaLogDirectory"] + "\\"; + string byteSize = "1073741824"; + + Utilities MyUtils = new Utilities(); + + // AMB1 - Job + string logOutputFileName_AMB1 = testName + "_AMB1.log"; + AMB_Settings AMB1 = new AMB_Settings + { + AMB_ServiceName = clientJobName, + AMB_PortAppReceives = "1000", + AMB_PortAMBSends = "1001", + AMB_ServiceLogPath = ambrosiaLogDir, + AMB_CreateService = "A", + AMB_PauseAtStart = "N", + AMB_PersistLogs = "Y", + AMB_NewLogTriggerSize = "1000", + AMB_ActiveActive = "N", + AMB_Version = "0" + }; + MyUtils.CallAMB(AMB1, logOutputFileName_AMB1, AMB_ModeConsts.RegisterInstance); + + int numShards = 3; + string[] logOutputFileName_AMB = new string[numShards]; + for (int i = 0; i < numShards; i++) + { + int shard_id = i + 1; + logOutputFileName_AMB[i] = testName + shard_id.ToString() + "_AMB.log"; + string portPrefix = (i + 2).ToString(); + AMB_Settings AMB_shard = new AMB_Settings + { + AMB_ServiceName = serverName, + AMB_PortAppReceives = portPrefix + "000", + AMB_PortAMBSends = portPrefix + "001", + AMB_ServiceLogPath = ambrosiaLogDir, + AMB_CreateService = "A", + AMB_PauseAtStart = "N", + AMB_PersistLogs = "Y", + AMB_NewLogTriggerSize = "1000", + AMB_ActiveActive = "N", + AMB_Version = "0", + AMB_ShardID = shard_id.ToString(), + }; + MyUtils.CallAMB(AMB_shard, logOutputFileName_AMB[i], AMB_ModeConsts.RegisterInstance); + } + + string logOutputFileName_ImmCoord1 = testName + "_ImmCoord1.log"; + int ImmCoordProcessID1 = MyUtils.StartImmCoord(clientJobName, 1500, logOutputFileName_ImmCoord1); + + int[] immCoordProcesses = new int[numShards]; + for (int i = 0; i < numShards; i++) + { + int shard_id = i + 1; + int port = (i + 2) * 1000 + 500; + string logOutputFileName_ImmCoord = testName + shard_id.ToString() + "_ImmCoord.log"; + immCoordProcesses[i] = MyUtils.StartImmCoord(serverName, port, logOutputFileName_ImmCoord); + } + + // Client Job Call + string logOutputFileName_ClientJob = testName + "_ClientJob.log"; + int clientJobProcessID = MyUtils.StartPerfClientJob("1001", "1000", clientJobName, serverName, "1024", "1", logOutputFileName_ClientJob); + + // Give it a few seconds to start + Thread.Sleep(2000); + + int[] serverProcesses = new int[numShards]; + string[] logOutputFileName_Server = new string[numShards]; + for (int i = 0; i < numShards; i++) + { + int shard_id = i + 1; + string portPrefix = (i + 2).ToString(); + logOutputFileName_Server[i] = testName + shard_id.ToString() + "_Server.log"; + serverProcesses[i] = MyUtils.StartPerfServer(portPrefix + "001", portPrefix + "000", clientJobName, serverName, logOutputFileName_Server[i], 1, false); + } + + // Delay until client is done - also check servers just to make sure + MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob, byteSize, 5, false, testName, true); // number of bytes processed + for (int i = 0; i < numShards; i++) + { + MyUtils.WaitForProcessToFinish(logOutputFileName_Server[i], byteSize, 5, false, testName, true); + } + + // Stop things so file is freed up and can be opened in verify + MyUtils.KillProcess(clientJobProcessID); + MyUtils.KillProcess(ImmCoordProcessID1); + for (int i = 0; i < numShards; i++) + { + MyUtils.KillProcess(immCoordProcesses[i]); + MyUtils.KillProcess(serverProcesses[i]); + } + + // Verify AMB + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_AMB1); + for (int i = 0; i < numShards; i++) + { + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_AMB[i]); + } + + // Verify Client + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_ClientJob); + + // Verify Server + for (int i = 0; i < numShards; i++) + { + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_Server[i]); + } + + // Verify integrity of Ambrosia logs by replaying + // TODO: Need to add shard logic + MyUtils.VerifyAmbrosiaLogFile(testName, Convert.ToInt64(byteSize), true, true, AMB1.AMB_Version); + }*/ + } +} diff --git a/AmbrosiaTest/AmbrosiaTest/Utilities.cs b/AmbrosiaTest/AmbrosiaTest/Utilities.cs index ff88bcd5..1e614271 100644 --- a/AmbrosiaTest/AmbrosiaTest/Utilities.cs +++ b/AmbrosiaTest/AmbrosiaTest/Utilities.cs @@ -28,6 +28,7 @@ public class AMB_Settings public string AMB_Version { get; set; } public string AMB_UpgradeToVersion { get; set; } public string AMB_ReplicaNumber { get; set; } + public string AMB_ShardID { get; set; } } @@ -460,7 +461,7 @@ public void VerifyTestOutputFileToCmpFile(string testOutputLogFile) // // Assumption: Test Output logs are .log and the cmp is the same file name but with .cmp extension //********************************************************************* - public void VerifyAmbrosiaLogFile(string testName, long numBytes, bool checkCmpFile, bool startWithFirstFile, string CurrentVersion, string optionalNumberOfClient = "", bool asyncTest = false) + public void VerifyAmbrosiaLogFile(string testName, long numBytes, bool checkCmpFile, bool startWithFirstFile, string CurrentVersion, string optionalNumberOfClient = "", bool asyncTest = false, long shardID = -1) { // Basically doing this for multi client stuff @@ -481,6 +482,11 @@ public void VerifyAmbrosiaLogFile(string testName, long numBytes, bool checkCmpF // used to get log file string ambrosiaClientLogDir = ConfigurationManager.AppSettings["AmbrosiaLogDirectory"] + "\\" + testName + "clientjob" + optionalMultiClientStartingPoint + "_" + CurrentVersion; string ambrosiaServerLogDir = ConfigurationManager.AppSettings["AmbrosiaLogDirectory"] + "\\" + testName + "server_" + CurrentVersion; + if (shardID != -1) + { + ambrosiaClientLogDir += "\\" + shardID.ToString(); + ambrosiaServerLogDir += "\\" + shardID.ToString(); + } string startingClientChkPtVersionNumber = "1"; string clientFirstFile = ""; @@ -576,8 +582,12 @@ public void VerifyAmbrosiaLogFile(string testName, long numBytes, bool checkCmpF AMB_Version = CurrentVersion.ToString(), AMB_TestingUpgrade = "N", AMB_PortAppReceives = "1000", - AMB_PortAMBSends = "1001" + AMB_PortAMBSends = "1001", }; + if (shardID != -1) + { + AMB1.AMB_ShardID = shardID.ToString(); + } CallAMB(AMB1, logOutputFileName_AMB1, AMB_ModeConsts.DebugInstance); // AMB for Server @@ -592,6 +602,10 @@ public void VerifyAmbrosiaLogFile(string testName, long numBytes, bool checkCmpF AMB_PortAppReceives = "2000", AMB_PortAMBSends = "2001" }; + if (shardID != -1) + { + AMB2.AMB_ShardID = shardID.ToString(); + } CallAMB(AMB2, logOutputFileName_AMB2, AMB_ModeConsts.DebugInstance); string logOutputFileName_ClientJob_Verify; @@ -634,7 +648,7 @@ public void VerifyAmbrosiaLogFile(string testName, long numBytes, bool checkCmpF } - public int StartImmCoord(string ImmCoordName, int portImmCoordListensAMB, string testOutputLogFile, bool ActiveActive=false, int replicaNum = 9999) + public int StartImmCoord(string ImmCoordName, int portImmCoordListensAMB, string testOutputLogFile, bool ActiveActive=false, int replicaNum = 9999, int shardID = -1) { // Launch the AMB process with these values @@ -647,6 +661,10 @@ public int StartImmCoord(string ImmCoordName, int portImmCoordListensAMB, string string argString = "-i=" + ImmCoordName + " -p=" + portImmCoordListensAMB.ToString(); + if (shardID != -1) + { + argString = argString + " -si=" + shardID.ToString(); + } // if Active Active then required to get replicanu if (ActiveActive) @@ -729,6 +747,9 @@ public void CallAMB(AMB_Settings AMBSettings, string testOutputLogFile, AMB_Mode if (AMBSettings.AMB_NewLogTriggerSize != null) argString = argString + " -lts=" + AMBSettings.AMB_NewLogTriggerSize; + if (AMBSettings.AMB_ShardID != null) + argString = argString + " -si=" + AMBSettings.AMB_ShardID; + break; case AMB_ModeConsts.AddReplica: @@ -789,6 +810,9 @@ public void CallAMB(AMB_Settings AMBSettings, string testOutputLogFile, AMB_Mode if (AMBSettings.AMB_TestingUpgrade != null && AMBSettings.AMB_TestingUpgrade != "N") argString = argString + " -tu"; + if (AMBSettings.AMB_ShardID != null) + argString = argString + " -si=" + AMBSettings.AMB_ShardID; + break; } From 60ff69e31aa345f20c54d77b1527bea32bb264e5 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Wed, 17 Jul 2019 16:34:47 -0700 Subject: [PATCH 15/22] Create a buffer append function. For sharding, we need to be able to concat output buffers to different records in the case a peer splits or merge. --- Ambrosia/Ambrosia/Program.cs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index e2b163a0..07d73fd5 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -692,6 +692,28 @@ internal IEnumerator GetEnumerator() return _bufferQ.GetEnumerator(); } + internal void Append(EventBuffer other) + { + AcquireTrimLock(2); + var bufferEnumerator = other.GetEnumerator(); + while (bufferEnumerator.MoveNext()) + { + var buffer = bufferEnumerator.Current; + var diff = buffer.HighestSeqNo - buffer.LowestSeqNo; + // Adjust sequence numbers + var writablePage = GetWritablePage(buffer.PageBytes.Length, buffer.LowestSeqNo); + writablePage.LowestSeqNo = buffer.LowestSeqNo; + writablePage.HighestSeqNo = buffer.HighestSeqNo; + writablePage.UnsentReplayableMessages += buffer.UnsentReplayableMessages; + writablePage.TotalReplayableMessages += buffer.TotalReplayableMessages; + // Copy the bytes into the page + writablePage.curLength += buffer.PageBytes.Length; + Buffer.BlockCopy(buffer.PageBytes, 0, writablePage.PageBytes, 0, buffer.PageBytes.Length); + ReleaseAppendLock(); + } + ReleaseTrimLock(); + } + internal async Task ReplayFromAsync(Stream outputStream, long firstSeqNo, bool reconnecting) From 256bcf0d51c3b06ac9ab45333fadb19cea69591c Mon Sep 17 00:00:00 2001 From: Jonathan Goldstein Date: Thu, 18 Jul 2019 17:34:42 -0700 Subject: [PATCH 16/22] Flushing output of c# client when taking becoming primary checkpoint --- Clients/CSharp/AmbrosiaLibCS/Immortal.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Clients/CSharp/AmbrosiaLibCS/Immortal.cs b/Clients/CSharp/AmbrosiaLibCS/Immortal.cs index b26198cf..3be43e38 100644 --- a/Clients/CSharp/AmbrosiaLibCS/Immortal.cs +++ b/Clients/CSharp/AmbrosiaLibCS/Immortal.cs @@ -65,6 +65,10 @@ public abstract class Immortal : IDisposable [CopyFromDeserializedImmortal] public SerializableCallCache CallCache = new SerializableCallCache(); + [DataMember] + [CopyFromDeserializedImmortal] + public int CommitID; + public SerializableCache TaskIdToSequenceNumber = new SerializableCache(); private ImmortalSerializerBase _immortalSerializer; @@ -395,7 +399,7 @@ protected async Task Dispatch(int bytesToRead = 0) #endif _cursor++; - await this.TakeCheckpointAsync(); + await this.TakeCheckpointAsync(true); this.IsPrimary = true; this.BecomingPrimary(); @@ -752,10 +756,15 @@ public async Task SaveTaskAsync() return true; } - private async Task TakeCheckpointAsync() + private async Task TakeCheckpointAsync(bool flushOutput = false) { // wait for quiesence _outputLock.Acquire(2); + if (flushOutput) + { + _ambrosiaSendToConnectionRecord.placeInOutput = + await _ambrosiaSendToConnectionRecord.BufferedOutput.SendAsync(_ambrosiaSendToStream, _ambrosiaSendToConnectionRecord.placeInOutput); + } _ambrosiaSendToConnectionRecord.BufferedOutput.LockOutputBuffer(); // Save current task state unless just resumed from a serialized task @@ -1387,6 +1396,8 @@ public void Start() else if (firstByte == AmbrosiaRuntime.takeCheckpointByte || firstByte == AmbrosiaRuntime.takeBecomingPrimaryCheckpointByte) { // Then this container is starting for the first time + // Set the commitID prior to taking the first checkpoint + MyImmortal.CommitID = commitID; if (firstByte == AmbrosiaRuntime.takeCheckpointByte) { #if DEBUG From 8a3d60514beb810f5948bcb1096e091da7ec3f78 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Mon, 22 Jul 2019 17:27:30 -0700 Subject: [PATCH 17/22] Fix _lastShuffleDest null bug It's possible for _lastShuffleDest to be null, which would result in a null dereference when we try to call length. --- Ambrosia/Ambrosia/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 07d73fd5..ecb54dac 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -3634,7 +3634,7 @@ private void ProcessRPC(FlexReadBuffer RpcBuffer) { // Find the appropriate connection record string destination; - if (_lastShuffleDest.Length < destBytesSize) + if (_lastShuffleDest == null || _lastShuffleDest.Length < destBytesSize) { _lastShuffleDest = new byte[destBytesSize]; } From 1c168a83678eb1ae7985ee08871c430e27cfcb04 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Mon, 22 Jul 2019 17:48:07 -0700 Subject: [PATCH 18/22] Update output connections --- Ambrosia/Ambrosia/Program.cs | 157 +++++++++++++++++--- AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs | 155 ++++++++++--------- AmbrosiaTest/AmbrosiaTest/Utilities.cs | 41 ++++- 3 files changed, 256 insertions(+), 97 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index ecb54dac..8139846b 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -3786,23 +3786,114 @@ private async Task SyncOutputConnectionAsync(ConcurrentD return outputConnectionRecord; } - private void SyncOutputs(string destString, ConcurrentDictionary>> ancestorsToIDs) + private OutputConnectionRecord GetOutputConnectionRecord(string destString) { - var destService = ParseServiceName(destString).Item1; + OutputConnectionRecord record; + lock (_outputs) + { + if (!_outputs.TryGetValue(destString, out record)) + { + // Set up the output record for the first time and add it to the dictionary + record = new OutputConnectionRecord(this); + _outputs[destString] = record; + Console.WriteLine("Adding output:{0}", destString); + } + else + { + Console.WriteLine("Restoring output:{0}", destString); + } + } + return record; + } + + private void MergeOutputConnections(OutputConnectionRecord baseConnection, + OutputConnectionRecord mergeConnection, + long lastProcessedReplayableID) + { + lock (baseConnection) + { + long firstSeq = -1; + long lastSeq = -1; + lock (mergeConnection) + { + mergeConnection.BufferedOutput.Trim(lastProcessedReplayableID, ref mergeConnection.placeInOutput); + firstSeq = baseConnection.LastSeqNoFromLocalService + 1; + lastSeq = mergeConnection.BufferedOutput.AdjustFirstSeqNoTo(firstSeq); + baseConnection.LastSeqNoFromLocalService = lastSeq; + baseConnection.ResettingConnection = true; + baseConnection.BufferedOutput.Append(mergeConnection.BufferedOutput); + baseConnection.ResettingConnection = false; + } + for (var outputSeq = firstSeq; outputSeq <= lastSeq; outputSeq++) + { + OutputRecordTracker tracker; + lock (_globalOutputTrackerLock) + { + _globalID += 1; + tracker = new OutputRecordTracker(_globalID, outputSeq); + _globalOutputTracker.Enqueue(tracker); + } + baseConnection.OutputTracker.Enqueue(tracker); + } + } + } + + private OutputConnectionRecord SetupOutputConnectionRecord(string destString, ConcurrentDictionary>> ancestorsToIDs) { + string destServiceBase = _serviceName; + if (!destString.Equals(ServiceName())) + { + destServiceBase = destString.Split('-')[0]; + } + OutputConnectionRecord record = GetOutputConnectionRecord(destString); foreach (var srcID in ancestorsToIDs.Keys) { + ConcurrentDictionary parentOutput; + if (_parentStates.ContainsKey(srcID)) + { + parentOutput = _parentStates[srcID].Outputs; + } + else + { + parentOutput = new ConcurrentDictionary(); + } foreach (var destID in ancestorsToIDs[srcID].Keys) { - var destName = ServiceName(destService, destID); - TrimOutput(destName, ancestorsToIDs[srcID][destID].Item1, ancestorsToIDs[srcID][destID].Item1); + var lastProcessedID = ancestorsToIDs[srcID][destID].Item1; + var lastProcessedReplayableID = ancestorsToIDs[srcID][destID].Item2; + string destService = DestinationShard(destServiceBase, destID); + + // Copy the parent output to the new output record + lock (parentOutput) + { + OutputConnectionRecord parentRecord; + if (parentOutput.TryRemove(destService, out parentRecord)) + { + MergeOutputConnections(record, parentRecord, lastProcessedReplayableID); + } + } + + // Copy old output to new output record + lock (_outputs) + { + OutputConnectionRecord oldRecord; + if (destService != destString && _outputs.TryRemove(destService, out oldRecord)) + { + MergeOutputConnections(record, oldRecord, lastProcessedReplayableID); + } + } } } + return record; } private async Task ToDataStreamAsync(Stream writeToStream, string destString, CancellationToken ct) { + if (destString.Equals(ServiceName())) + { + destString = ""; + } if (_sharded) { Serializer.SerializeAncestorMessage(writeToStream, ancestorByte, _ancestors[ServiceName()]); @@ -3812,14 +3903,14 @@ private async Task ToDataStreamAsync(Stream writeToStream, // Get the seqNo of the replay/filter point var commitSeqNo = result.Item1; var commitSeqNoReplayable = result.Item2; + var ancestorsToIDs = new ConcurrentDictionary>>(); if (_sharded) { - var ancestorsToIDs = result.Item3; - SyncOutputs(destString, ancestorsToIDs); + ancestorsToIDs = result.Item3; } - OutputConnectionRecord outputConnectionRecord = await SyncOutputConnectionAsync(_outputs, destString, commitSeqNo, commitSeqNoReplayable); + var outputConnectionRecord = SetupOutputConnectionRecord(destString, ancestorsToIDs); if (_sharded) { Serializer.SerializeShardTrimMessage(writeToStream, shardTrimByte); @@ -3827,6 +3918,42 @@ private async Task ToDataStreamAsync(Stream writeToStream, try { + // Reset the output cursor if it exists + outputConnectionRecord.BufferedOutput.AcquireTrimLock(2); + outputConnectionRecord.placeInOutput = new EventBuffer.BuffersCursor(null, -1, 0); + outputConnectionRecord.BufferedOutput.ReleaseTrimLock(); + if (outputConnectionRecord.ConnectingAfterRestart) + { + // We've been through recovery (at least partially), and have scrubbed all ephemeral calls. Must now rebase + // seq nos using the markers which were sent by the listener. Must first take locks to ensure no interference + lock (outputConnectionRecord) + { + // Don't think I actually need this lock, but can't hurt and shouldn't affect perf. + outputConnectionRecord.BufferedOutput.AcquireTrimLock(2); + outputConnectionRecord.BufferedOutput.RebaseSeqNosInBuffer(commitSeqNo, commitSeqNoReplayable); + outputConnectionRecord.LastSeqNoFromLocalService += commitSeqNo - commitSeqNoReplayable; + outputConnectionRecord.ConnectingAfterRestart = false; + outputConnectionRecord.BufferedOutput.ReleaseTrimLock(); + } + } + // If recovering, make sure event replay will be filtered out + outputConnectionRecord.ReplayFrom = commitSeqNo; + + if (outputConnectionRecord.WillResetConnection) + { + // Register our immediate intent to set the connection. This unblocks output writers + outputConnectionRecord.ResettingConnection = true; + // This lock avoids interference with buffering RPCs + lock (outputConnectionRecord) + { + // If first reconnect/connect after reset, simply adjust the seq no for the first sent message to the received commit seq no + outputConnectionRecord.ResettingConnection = false; + outputConnectionRecord.LastSeqNoFromLocalService = outputConnectionRecord.BufferedOutput.AdjustFirstSeqNoTo(commitSeqNo); + outputConnectionRecord.WillResetConnection = false; + } + } + outputConnectionRecord.LastSeqSentToReceiver = commitSeqNo - 1; + // Enqueue a replay send if (outputConnectionRecord._sendsEnqueued == 0) { @@ -3885,25 +4012,11 @@ private async Task ToControlStreamAsync(Stream writeToStream, CancellationToken ct) { - OutputConnectionRecord outputConnectionRecord; if (destString.Equals(ServiceName())) { destString = ""; } - lock (_outputs) - { - if (!_outputs.TryGetValue(destString, out outputConnectionRecord)) - { - // Set up the output record for the first time and add it to the dictionary - outputConnectionRecord = new OutputConnectionRecord(this); - _outputs[destString] = outputConnectionRecord; - Console.WriteLine("Adding output:{0}", destString); - } - else - { - Console.WriteLine("restoring output:{0}", destString); - } - } + OutputConnectionRecord outputConnectionRecord = GetOutputConnectionRecord(destString); // Process remote trim message var inputFlexBuffer = new FlexReadBuffer(); await FlexReadBuffer.DeserializeAsync(writeToStream, inputFlexBuffer, ct); diff --git a/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs b/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs index 480f5d49..4e277217 100644 --- a/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs +++ b/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs @@ -2,6 +2,7 @@ using System; using System.Configuration; using System.Threading; +using System.Windows.Forms; namespace AmbrosiaTest { @@ -109,12 +110,11 @@ public void Shard_UnitTest_BasicEndtoEnd_Test() // Verify integrity of Ambrosia logs by replaying MyUtils.VerifyAmbrosiaLogFile(testName, Convert.ToInt64(byteSize), true, true, AMB1.AMB_Version, shardID: 1); } -/* [TestMethod] - public void Shard_UnitTest_MultiShardEndToEnd_Test() + public void Shard_UnitTest_SingleReshardEndtoEnd_Test() { - // Test that multi-shards per server and client behaves the same as the single shard case - string testName = "shardunitendtoendtest"; + // Test that one shard per server and client works + string testName = "shardunitsinglereshardendtoendtest"; string clientJobName = testName + "clientjob"; string serverName = testName + "server"; string ambrosiaLogDir = ConfigurationManager.AppSettings["AmbrosiaLogDirectory"] + "\\"; @@ -135,98 +135,107 @@ public void Shard_UnitTest_MultiShardEndToEnd_Test() AMB_PersistLogs = "Y", AMB_NewLogTriggerSize = "1000", AMB_ActiveActive = "N", - AMB_Version = "0" + AMB_Version = "0", + AMB_ShardID = "1", }; MyUtils.CallAMB(AMB1, logOutputFileName_AMB1, AMB_ModeConsts.RegisterInstance); - int numShards = 3; - string[] logOutputFileName_AMB = new string[numShards]; - for (int i = 0; i < numShards; i++) + // AMB2 - Shard 1 + string logOutputFileName_AMB2 = testName + "_AMB2.log"; + AMB_Settings AMB2 = new AMB_Settings { - int shard_id = i + 1; - logOutputFileName_AMB[i] = testName + shard_id.ToString() + "_AMB.log"; - string portPrefix = (i + 2).ToString(); - AMB_Settings AMB_shard = new AMB_Settings - { - AMB_ServiceName = serverName, - AMB_PortAppReceives = portPrefix + "000", - AMB_PortAMBSends = portPrefix + "001", - AMB_ServiceLogPath = ambrosiaLogDir, - AMB_CreateService = "A", - AMB_PauseAtStart = "N", - AMB_PersistLogs = "Y", - AMB_NewLogTriggerSize = "1000", - AMB_ActiveActive = "N", - AMB_Version = "0", - AMB_ShardID = shard_id.ToString(), - }; - MyUtils.CallAMB(AMB_shard, logOutputFileName_AMB[i], AMB_ModeConsts.RegisterInstance); - } + AMB_ServiceName = serverName, + AMB_PortAppReceives = "2000", + AMB_PortAMBSends = "2001", + AMB_ServiceLogPath = ambrosiaLogDir, + AMB_CreateService = "A", + AMB_PauseAtStart = "N", + AMB_PersistLogs = "Y", + AMB_NewLogTriggerSize = "1000", + AMB_ActiveActive = "N", + AMB_Version = "0", + AMB_ShardID = "1", + }; + MyUtils.CallAMB(AMB2, logOutputFileName_AMB2, AMB_ModeConsts.RegisterInstance); + // AMB 3 - Shard 2 + string logOutputFileName_AMB3 = testName + "_AMB3.log"; + AMB_Settings AMB3 = new AMB_Settings + { + AMB_ServiceName = serverName, + AMB_PortAppReceives = "3000", + AMB_PortAMBSends = "3001", + AMB_ServiceLogPath = ambrosiaLogDir, + AMB_CreateService = "A", + AMB_PauseAtStart = "N", + AMB_PersistLogs = "Y", + AMB_NewLogTriggerSize = "1000", + AMB_ActiveActive = "N", + AMB_Version = "0", + AMB_ShardID = "2", + AMB_OldShards = "1", + AMB_NewShards = "2" + }; + MyUtils.CallAMB(AMB3, logOutputFileName_AMB3, AMB_ModeConsts.AddShard); + + // ImmCoord1 string logOutputFileName_ImmCoord1 = testName + "_ImmCoord1.log"; - int ImmCoordProcessID1 = MyUtils.StartImmCoord(clientJobName, 1500, logOutputFileName_ImmCoord1); + int ImmCoordProcessID1 = MyUtils.StartImmCoord(clientJobName, 1500, logOutputFileName_ImmCoord1, shardID: 1); - int[] immCoordProcesses = new int[numShards]; - for (int i = 0; i < numShards; i++) - { - int shard_id = i + 1; - int port = (i + 2) * 1000 + 500; - string logOutputFileName_ImmCoord = testName + shard_id.ToString() + "_ImmCoord.log"; - immCoordProcesses[i] = MyUtils.StartImmCoord(serverName, port, logOutputFileName_ImmCoord); - } + // ImmCoord2 + string logOutputFileName_ImmCoord2 = testName + "_ImmCoord2.log"; + int ImmCoordProcessID2 = MyUtils.StartImmCoord(serverName, 2500, logOutputFileName_ImmCoord2, shardID: 1); - // Client Job Call + // Client string logOutputFileName_ClientJob = testName + "_ClientJob.log"; int clientJobProcessID = MyUtils.StartPerfClientJob("1001", "1000", clientJobName, serverName, "1024", "1", logOutputFileName_ClientJob); // Give it a few seconds to start Thread.Sleep(2000); - int[] serverProcesses = new int[numShards]; - string[] logOutputFileName_Server = new string[numShards]; - for (int i = 0; i < numShards; i++) - { - int shard_id = i + 1; - string portPrefix = (i + 2).ToString(); - logOutputFileName_Server[i] = testName + shard_id.ToString() + "_Server.log"; - serverProcesses[i] = MyUtils.StartPerfServer(portPrefix + "001", portPrefix + "000", clientJobName, serverName, logOutputFileName_Server[i], 1, false); - } - - // Delay until client is done - also check servers just to make sure - MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob, byteSize, 5, false, testName, true); // number of bytes processed - for (int i = 0; i < numShards; i++) - { - MyUtils.WaitForProcessToFinish(logOutputFileName_Server[i], byteSize, 5, false, testName, true); - } + // First Server Call + string logOutputFileName_Server1 = testName + "_Server1.log"; + int serverProcessID1 = MyUtils.StartPerfServer("2001", "2000", clientJobName, serverName, logOutputFileName_Server1, 1, false); + + // Delay until client is done - also check Server just to make sure + //bool pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob, byteSize, 5, false, testName, true); // Number of bytes processed + //pass = MyUtils.WaitForProcessToFinish(logOutputFileName_Server1, byteSize, 5, false, testName, true); - // Stop things so file is freed up and can be opened in verify + // Give it 2 seconds to do something before killing it + //Thread.Sleep(2000); + Application.DoEvents(); // if don't do this ... system sees thread as blocked thread and throws message. + + // ImmCoord3 + string logOutputFileName_ImmCoord3 = testName + "_ImmCoord3.log"; + int ImmCoordProcessID3 = MyUtils.StartImmCoord(serverName, 3500, logOutputFileName_ImmCoord3, shardID: 2); + + // Second Server Call + string logOutputFileName_Server2 = testName + "_Server2.log"; + int serverProcessID2 = MyUtils.StartPerfServer("3001", "3000", clientJobName, serverName, logOutputFileName_Server2, 1, false); + bool pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob, byteSize, 10, false, testName, true); // Number of bytes processed + pass = MyUtils.WaitForProcessToFinish(logOutputFileName_Server2, byteSize, 10, false, testName, true); + + // Stop things to file is freed up and can be opened in verify MyUtils.KillProcess(clientJobProcessID); + MyUtils.KillProcess(serverProcessID1); + MyUtils.KillProcess(serverProcessID2); MyUtils.KillProcess(ImmCoordProcessID1); - for (int i = 0; i < numShards; i++) - { - MyUtils.KillProcess(immCoordProcesses[i]); - MyUtils.KillProcess(serverProcesses[i]); - } + MyUtils.KillProcess(ImmCoordProcessID2); + MyUtils.KillProcess(ImmCoordProcessID3); // Verify AMB MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_AMB1); - for (int i = 0; i < numShards; i++) - { - MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_AMB[i]); - } + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_AMB2); + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_AMB3); // Verify Client MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_ClientJob); - - // Verify Server - for (int i = 0; i < numShards; i++) - { - MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_Server[i]); - } - + + // Verify Server 1 + MyUtils.VerifyTestOutputFileToCmpFile(logOutputFileName_Server2); + /* // Verify integrity of Ambrosia logs by replaying - // TODO: Need to add shard logic - MyUtils.VerifyAmbrosiaLogFile(testName, Convert.ToInt64(byteSize), true, true, AMB1.AMB_Version); - }*/ + MyUtils.VerifyAmbrosiaLogFile(testName, Convert.ToInt64(byteSize), true, true, AMB1.AMB_Version, shardID: 1);*/ + } } } diff --git a/AmbrosiaTest/AmbrosiaTest/Utilities.cs b/AmbrosiaTest/AmbrosiaTest/Utilities.cs index 1e614271..4e25a955 100644 --- a/AmbrosiaTest/AmbrosiaTest/Utilities.cs +++ b/AmbrosiaTest/AmbrosiaTest/Utilities.cs @@ -29,11 +29,12 @@ public class AMB_Settings public string AMB_UpgradeToVersion { get; set; } public string AMB_ReplicaNumber { get; set; } public string AMB_ShardID { get; set; } - + public string AMB_OldShards { get; set; } + public string AMB_NewShards { get; set; } } // These are the different modes of what the AMB is called - public enum AMB_ModeConsts { RegisterInstance, AddReplica, DebugInstance }; + public enum AMB_ModeConsts { RegisterInstance, AddReplica, DebugInstance, AddShard }; public class Utilities { @@ -814,6 +815,42 @@ public void CallAMB(AMB_Settings AMBSettings, string testOutputLogFile, AMB_Mode argString = argString + " -si=" + AMBSettings.AMB_ShardID; break; + + case AMB_ModeConsts.AddShard: + argString = "AddShard " + "-si=" + AMBSettings.AMB_ShardID + " -i=" + AMBSettings.AMB_ServiceName + + " -rp=" + AMBSettings.AMB_PortAppReceives + " -sp=" + AMBSettings.AMB_PortAMBSends + + " -r=" + AMBSettings.AMB_ReplicaNumber + " -os=" + AMBSettings.AMB_OldShards + + " -ns=" + AMBSettings.AMB_NewShards; + + // add Service log path + if (AMBSettings.AMB_ServiceLogPath != null) + argString = argString + " -l=" + AMBSettings.AMB_ServiceLogPath; + + // add pause at start + if (AMBSettings.AMB_PauseAtStart != null && AMBSettings.AMB_PauseAtStart != "N") + argString = argString + " -ps"; + + // add no persist logs at start + if (AMBSettings.AMB_PersistLogs != null && AMBSettings.AMB_PersistLogs != "Y") + argString = argString + " -npl"; + + // add new log trigger size if it exists + if (AMBSettings.AMB_NewLogTriggerSize != null) + argString = argString + " -lts=" + AMBSettings.AMB_NewLogTriggerSize; + + // add active active + if (AMBSettings.AMB_ActiveActive != null && AMBSettings.AMB_ActiveActive != "N") + argString = argString + " -aa"; + + // add current version if it exists + if (AMBSettings.AMB_Version != null) + argString = argString + " -cv=" + AMBSettings.AMB_Version; + + // add upgrade version if it exists + if (AMBSettings.AMB_UpgradeToVersion != null) + argString = argString + " -uv=" + AMBSettings.AMB_UpgradeToVersion; + + break; } int processID = LaunchProcess(workingDir, fileNameExe, argString, false, testOutputLogFile); From 3aaf99509e05356d98c8c73c87352044638f46fa Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Mon, 22 Jul 2019 17:49:29 -0700 Subject: [PATCH 19/22] Special case sharded case in ProcessRPC We don't want to update _outputs when a shard is launching. There will be an output variable for each parent that will be accessed in parallel. We need to update this variable to ensure recovery state is correct and to prevent race conditions. --- Ambrosia/Ambrosia/Program.cs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 8139846b..dff184f2 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -3648,8 +3648,27 @@ private void ProcessRPC(FlexReadBuffer RpcBuffer) // During replay, the output connection won't exist if this is the first message ever and no trim record has been processed yet. if (!_outputs.TryGetValue(destination, out _shuffleOutputRecord)) { - _shuffleOutputRecord = new OutputConnectionRecord(this); - _outputs[destination] = _shuffleOutputRecord; + if (_oldShards.Length > 0) + { + // TODO: We're still replaying from potentially multiple shards. + // For now, we assume there is only one parent shard, but we need to update + // the client code to pass which shard sent this during recovery. + Debug.Assert(_parentStates.Count == 1); + MachineState parent = _parentStates.Values.First(); + + if (!parent.Outputs.TryGetValue(destination, out _shuffleOutputRecord)) + { + _shuffleOutputRecord = new OutputConnectionRecord(this); + _outputs[destination] = _shuffleOutputRecord; + } else + { + _lastShuffleDest = null; + } + } else + { + _shuffleOutputRecord = new OutputConnectionRecord(this); + _outputs[destination] = _shuffleOutputRecord; + } } } } From 7a44f3d3c17e80320fc0b22509d3f574d6b97b8a Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Tue, 23 Jul 2019 15:26:22 -0700 Subject: [PATCH 20/22] Incorporate CRA sharded API --- Ambrosia/Ambrosia/Program.cs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index dff184f2..05319ca6 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -4733,6 +4733,24 @@ class Program private static string _oldShards = ""; private static string _newShards = ""; + private static void InstantiateVertex(CRAClientLibrary client, string instanceName, string vertexName, string vertexDefinition, object vertexParameter, bool sharded) + { + CRAErrorCode result; + if (!sharded) + { + result = client.InstantiateVertex(instanceName, vertexName, vertexDefinition, vertexParameter); + } else + { + ConcurrentDictionary vertexShards = new ConcurrentDictionary(); + vertexShards[instanceName] = 1; + result = client.InstantiateShardedVertex(vertexName, vertexDefinition, vertexParameter, vertexShards); + } + if (result != CRAErrorCode.Success) + { + throw new Exception(); + } + } + static void Main(string[] args) { ParseAndValidateOptions(args); @@ -4791,10 +4809,7 @@ static void Main(string[] args) serializedParams = textWriter.ToString(); } - if (client.InstantiateVertex(replicaName, shardName, param.AmbrosiaBinariesLocation, serializedParams) != CRAErrorCode.Success) - { - throw new Exception(); - } + InstantiateVertex(client, replicaName, shardName, param.AmbrosiaBinariesLocation, serializedParams, _shardID > 0); client.AddEndpoint(shardName, AmbrosiaRuntime.AmbrosiaDataInputsName, true, true); client.AddEndpoint(shardName, AmbrosiaRuntime.AmbrosiaDataOutputsName, false, true); From 74da14311869166d819f30b9869c953580549744 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Tue, 23 Jul 2019 17:28:45 -0700 Subject: [PATCH 21/22] Doing sharding stuff --- Ambrosia/Ambrosia/Program.cs | 93 ++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 40 deletions(-) diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 05319ca6..341798a2 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -718,39 +718,39 @@ internal async Task ReplayFromAsync(Stream outputStream, long firstSeqNo, bool reconnecting) { -/* if (reconnecting) - { - var bufferE = _bufferQ.GetEnumerator(); - while (bufferE.MoveNext()) - { - var curBuffer = bufferE.Current; - Debug.Assert(curBuffer.LowestSeqNo <= firstSeqNo); - int skipEvents = 0; - if (curBuffer.HighestSeqNo >= firstSeqNo) - { - // We need to send some or all of this buffer - skipEvents = (int)(Math.Max(0, firstSeqNo - curBuffer.LowestSeqNo)); - } - else - { - skipEvents = 0; - } - int bufferPos = 0; - AcquireAppendLock(2); - curBuffer.UnsentReplayableMessages = curBuffer.TotalReplayableMessages; - for (int i = 0; i < skipEvents; i++) - { - int eventSize = curBuffer.PageBytes.ReadBufferedInt(bufferPos); - var methodID = curBuffer.PageBytes.ReadBufferedInt(bufferPos + StreamCommunicator.IntSize(eventSize) + 2); - if (curBuffer.PageBytes[bufferPos + StreamCommunicator.IntSize(eventSize) + 2 + StreamCommunicator.IntSize(methodID)] != (byte)RpcTypes.RpcType.Impulse) + /* if (reconnecting) { - curBuffer.UnsentReplayableMessages--; - } - bufferPos += eventSize + StreamCommunicator.IntSize(eventSize); - } - ReleaseAppendLock(); - } - }*/ + var bufferE = _bufferQ.GetEnumerator(); + while (bufferE.MoveNext()) + { + var curBuffer = bufferE.Current; + Debug.Assert(curBuffer.LowestSeqNo <= firstSeqNo); + int skipEvents = 0; + if (curBuffer.HighestSeqNo >= firstSeqNo) + { + // We need to send some or all of this buffer + skipEvents = (int)(Math.Max(0, firstSeqNo - curBuffer.LowestSeqNo)); + } + else + { + skipEvents = 0; + } + int bufferPos = 0; + AcquireAppendLock(2); + curBuffer.UnsentReplayableMessages = curBuffer.TotalReplayableMessages; + for (int i = 0; i < skipEvents; i++) + { + int eventSize = curBuffer.PageBytes.ReadBufferedInt(bufferPos); + var methodID = curBuffer.PageBytes.ReadBufferedInt(bufferPos + StreamCommunicator.IntSize(eventSize) + 2); + if (curBuffer.PageBytes[bufferPos + StreamCommunicator.IntSize(eventSize) + 2 + StreamCommunicator.IntSize(methodID)] != (byte)RpcTypes.RpcType.Impulse) + { + curBuffer.UnsentReplayableMessages--; + } + bufferPos += eventSize + StreamCommunicator.IntSize(eventSize); + } + ReleaseAppendLock(); + } + }*/ var bufferEnumerator = _bufferQ.GetEnumerator(); // Scan through pages from head to tail looking for events to output while (bufferEnumerator.MoveNext()) @@ -764,7 +764,7 @@ internal async Task ReplayFromAsync(Stream outputStream, int bufferPos = 0; if (true) // BUGBUG We are temporarily disabling this optimization which avoids unnecessary locking as reconnecting is not a sufficient criteria: We found a case where input is arriving during reconnection where counting was getting disabled incorrectly. Further investigation is required. -// if (reconnecting) // BUGBUG We are temporarily disabling this optimization which avoids unnecessary locking as reconnecting is not a sufficient criteria: We found a case where input is arriving during reconnection where counting was getting disabled incorrectly. Further investigation is required. + // if (reconnecting) // BUGBUG We are temporarily disabling this optimization which avoids unnecessary locking as reconnecting is not a sufficient criteria: We found a case where input is arriving during reconnection where counting was getting disabled incorrectly. Further investigation is required. { // We need to reset how many replayable messages have been sent. We want to minimize the use of // this codepath because of the expensive locking, which can compete with new RPCs getting appended @@ -1202,6 +1202,7 @@ public static class AmbrosiaRuntimeParms public static bool _looseAttach = false; } + public class AmbrosiaRuntime : VertexBase { #if _WINDOWS @@ -1511,6 +1512,7 @@ private async Task Commit(byte[] buf, } catch (Exception e) { + Console.WriteLine(e.StackTrace); _myAmbrosia.OnError(5, e.Message); } _bufbak = buf; @@ -2648,7 +2650,7 @@ private void AddParentStates(MachineState[] states) foreach (var state in states) { AddParentInputState(state); - AddParentOutputState(state); + //AddParentOutputState(state); } } @@ -4415,7 +4417,6 @@ private void TrimGlobalOutputTracker() { _globalOutputTracker.TryDequeue(out tracker); tracker = _globalOutputTracker.First(); - } } catch (InvalidOperationException) @@ -4428,7 +4429,6 @@ private void TrimGlobalOutputTracker() public void TrimOutput(string key, long lastProcessedID, long lastProcessedReplayableID) { - Console.WriteLine("Trimming {0} up to ({1}, {2})", key, lastProcessedID, lastProcessedReplayableID); OutputConnectionRecord outputConnectionRecord; if (!_outputs.TryGetValue(key, out outputConnectionRecord)) { @@ -4751,6 +4751,23 @@ private static void InstantiateVertex(CRAClientLibrary client, string instanceNa } } + /* private static void DefineVertex(CRAClientLibrary client, string vertexDefinition, bool sharded) + { + CRAErrorCode result; + if (!sharded) + { + result = client.DefineVertex(param.AmbrosiaBinariesLocation, () => new AmbrosiaRuntime()); + } + /* if (client.DefineVertex(param.AmbrosiaBinariesLocation, () => new AmbrosiaRuntime()) != CRAErrorCode.Success) + { + throw new Exception(); + } + if (result != CRAErrorCode.Success) + { + throw new Exception(); + } + }(*/ + static void Main(string[] args) { ParseAndValidateOptions(args); @@ -4796,10 +4813,6 @@ static void Main(string[] args) try { - if (client.DefineVertex(param.AmbrosiaBinariesLocation, () => new AmbrosiaRuntime()) != CRAErrorCode.Success) - { - throw new Exception(); - } // Workaround because of limitation in parameter serialization in CRA XmlSerializer xmlSerializer = new XmlSerializer(param.GetType()); string serializedParams; From 70f1694cab6d3233d677a5909979d8781b8e34a0 Mon Sep 17 00:00:00 2001 From: Shannon Joyner Date: Fri, 23 Aug 2019 16:13:37 -0700 Subject: [PATCH 22/22] Debugging --- Ambrosia/Ambrosia.sln | 32 ++-- Ambrosia/Ambrosia/Program.cs | 172 ++++++++++++++------ AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs | 2 +- AmbrosiaTest/AmbrosiaTest/app.config | 2 +- Samples/HelloWorld/Client1/Program.cs | 16 +- Samples/HelloWorld/Client2/Program.cs | 40 ++++- Samples/HelloWorld/Server/Program.cs | 10 +- 7 files changed, 198 insertions(+), 76 deletions(-) diff --git a/Ambrosia/Ambrosia.sln b/Ambrosia/Ambrosia.sln index 1704d0d7..da4f809f 100644 --- a/Ambrosia/Ambrosia.sln +++ b/Ambrosia/Ambrosia.sln @@ -3,47 +3,45 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 VisualStudioVersion = 15.0.27004.2006 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "adv-file-ops", "adv-file-ops\adv-file-ops.vcxproj", "{5852AC33-6B01-44F5-BAF3-2AAF796E8449}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0BEADEF6-C937-465D-814B-726C3E2A22BA}" ProjectSection(SolutionItems) = preProject nuget.config = nuget.config EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ImmortalCoordinator", "..\ImmortalCoordinator\ImmortalCoordinator.csproj", "{5C94C516-377C-4113-8C5F-DF4A016D1B3A}" - ProjectSection(ProjectDependencies) = postProject - {5852AC33-6B01-44F5-BAF3-2AAF796E8449} = {5852AC33-6B01-44F5-BAF3-2AAF796E8449} - EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Ambrosia", "Ambrosia\Ambrosia.csproj", "{F704AE0A-C37B-4D30-B9ED-0C76C62D66EC}" - ProjectSection(ProjectDependencies) = postProject - {5852AC33-6B01-44F5-BAF3-2AAF796E8449} = {5852AC33-6B01-44F5-BAF3-2AAF796E8449} - EndProjectSection EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AmbrosiaTests", "AmbrosiaTests\AmbrosiaTests.csproj", "{4339AF42-4A49-42E9-8571-0DC4CEB0D1CB}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CRA.ClientLibrary", "..\CRA\src\CRA.ClientLibrary\CRA.ClientLibrary.csproj", "{D1198E24-4E02-4586-832A-14E6035009B0}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU Debug|x64 = Debug|x64 + Release|Any CPU = Release|Any CPU Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {5852AC33-6B01-44F5-BAF3-2AAF796E8449}.Debug|x64.ActiveCfg = Release|x64 - {5852AC33-6B01-44F5-BAF3-2AAF796E8449}.Debug|x64.Build.0 = Release|x64 - {5852AC33-6B01-44F5-BAF3-2AAF796E8449}.Release|x64.ActiveCfg = Release|x64 - {5852AC33-6B01-44F5-BAF3-2AAF796E8449}.Release|x64.Build.0 = Release|x64 + {5C94C516-377C-4113-8C5F-DF4A016D1B3A}.Debug|Any CPU.ActiveCfg = Debug|x64 {5C94C516-377C-4113-8C5F-DF4A016D1B3A}.Debug|x64.ActiveCfg = Debug|x64 {5C94C516-377C-4113-8C5F-DF4A016D1B3A}.Debug|x64.Build.0 = Debug|x64 + {5C94C516-377C-4113-8C5F-DF4A016D1B3A}.Release|Any CPU.ActiveCfg = Release|x64 {5C94C516-377C-4113-8C5F-DF4A016D1B3A}.Release|x64.ActiveCfg = Release|x64 {5C94C516-377C-4113-8C5F-DF4A016D1B3A}.Release|x64.Build.0 = Release|x64 + {F704AE0A-C37B-4D30-B9ED-0C76C62D66EC}.Debug|Any CPU.ActiveCfg = Debug|x64 {F704AE0A-C37B-4D30-B9ED-0C76C62D66EC}.Debug|x64.ActiveCfg = Debug|x64 {F704AE0A-C37B-4D30-B9ED-0C76C62D66EC}.Debug|x64.Build.0 = Debug|x64 + {F704AE0A-C37B-4D30-B9ED-0C76C62D66EC}.Release|Any CPU.ActiveCfg = Release|x64 {F704AE0A-C37B-4D30-B9ED-0C76C62D66EC}.Release|x64.ActiveCfg = Release|x64 {F704AE0A-C37B-4D30-B9ED-0C76C62D66EC}.Release|x64.Build.0 = Release|x64 - {4339AF42-4A49-42E9-8571-0DC4CEB0D1CB}.Debug|x64.ActiveCfg = Debug|x64 - {4339AF42-4A49-42E9-8571-0DC4CEB0D1CB}.Debug|x64.Build.0 = Debug|x64 - {4339AF42-4A49-42E9-8571-0DC4CEB0D1CB}.Release|x64.ActiveCfg = Release|x64 - {4339AF42-4A49-42E9-8571-0DC4CEB0D1CB}.Release|x64.Build.0 = Release|x64 + {D1198E24-4E02-4586-832A-14E6035009B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D1198E24-4E02-4586-832A-14E6035009B0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D1198E24-4E02-4586-832A-14E6035009B0}.Debug|x64.ActiveCfg = Debug|Any CPU + {D1198E24-4E02-4586-832A-14E6035009B0}.Debug|x64.Build.0 = Debug|Any CPU + {D1198E24-4E02-4586-832A-14E6035009B0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D1198E24-4E02-4586-832A-14E6035009B0}.Release|Any CPU.Build.0 = Release|Any CPU + {D1198E24-4E02-4586-832A-14E6035009B0}.Release|x64.ActiveCfg = Release|Any CPU + {D1198E24-4E02-4586-832A-14E6035009B0}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Ambrosia/Ambrosia/Program.cs b/Ambrosia/Ambrosia/Program.cs index 341798a2..0fd54471 100644 --- a/Ambrosia/Ambrosia/Program.cs +++ b/Ambrosia/Ambrosia/Program.cs @@ -155,7 +155,7 @@ internal static void AmbrosiaSerialize(this ConcurrentDictionary AmbrosiaDeserialize(this ConcurrentDictionary dict, LogReader readFromStream) + internal static ConcurrentDictionary AmbrosiaDeserialize(this ConcurrentDictionary dict, LogReader readFromStream, string serviceName) { var _retVal = new ConcurrentDictionary(); var dictCount = readFromStream.ReadIntFixed(); @@ -212,7 +212,7 @@ internal static void AmbrosiaSerialize(this ConcurrentDictionary AmbrosiaDeserialize(this ConcurrentDictionary dict, LogReader readFromStream, AmbrosiaRuntime thisAmbrosia) + internal static ConcurrentDictionary AmbrosiaDeserialize(this ConcurrentDictionary dict, LogReader readFromStream, AmbrosiaRuntime thisAmbrosia, string serviceName) { var _retVal = new ConcurrentDictionary(); var dictCount = readFromStream.ReadIntFixed(); @@ -2184,6 +2184,7 @@ public async Task FromStreamAsync(Stream stream, string otherProcess, string oth internal long _globalID = 0; ConcurrentDictionary _inputs; ConcurrentDictionary _outputs; + ConcurrentDictionary _serviceNames = new ConcurrentDictionary(); // Input / output state of parent shards. This is needed for recovery. ConcurrentDictionary _parentStates = new ConcurrentDictionary(); ConcurrentDictionary _ancestors; @@ -2279,6 +2280,7 @@ internal enum AARole { Primary, Secondary, Checkpointer, ReshardSecondary }; internal void OnError(int ErrNo, string ErrorMessage) { Console.WriteLine("FATAL ERROR " + ErrNo.ToString() + ": " + ErrorMessage); + Console.WriteLine("OH NO {0}", (DateTime.Now - new DateTime(1970, 1, 1)).TotalMilliseconds); Console.Out.Flush(); Console.Out.Flush(); _coral.KillLocalWorker(""); @@ -2439,8 +2441,14 @@ private async Task RecoverOrStartAsync(long checkpointToLoad = -1, Recovering = false; PrepareToRecoverOrStart(); + if (_shardID < 3) + { + _createService = true; + } + if (!_runningRepro) { + Console.WriteLine("CHECKPOINT " + CheckpointName(_lastCommittedCheckpoint)); RuntimeChecksOnProcessStart(); } @@ -2451,13 +2459,17 @@ private async Task RecoverOrStartAsync(long checkpointToLoad = -1, } // Determine if we are recovering - if (!_createService) + if (ServiceName() == "server-3") { Recovering = true; _restartWithRecovery = true; if (_oldShards.Length > 0) { + Console.WriteLine("HERE"); + var start = DateTime.Now; RecoverFromShards(checkpointToLoad, testUpgrade); + var end = DateTime.Now; + Console.WriteLine("RecoverFromShards {0}", end - start); } else { MachineState state = new MachineState(_shardID); @@ -2516,9 +2528,9 @@ private async Task RecoverAsync(MachineState state, long checkpointToLoad = -1, // Recover committer state.Committer = new Committer(_localServiceSendToStream, _persistLogs, this, -1, checkpointStream); // Recover input connections - state.Inputs = state.Inputs.AmbrosiaDeserialize(checkpointStream); + state.Inputs = state.Inputs.AmbrosiaDeserialize(checkpointStream, ServiceName()); // Recover output connections - state.Outputs = state.Outputs.AmbrosiaDeserialize(checkpointStream, this); + state.Outputs = state.Outputs.AmbrosiaDeserialize(checkpointStream, this, ServiceName()); UnbufferNonreplayableCalls(state.Outputs); // Restore new service from checkpoint var serviceCheckpoint = new FlexReadBuffer(); @@ -2572,7 +2584,7 @@ private async Task PrepareToBecomePrimaryAsync() await MoveServiceToNextLogFileAsync(); } - if (_activeActive || _shardID > 0) + if (ServiceName() == "server-2" || ServiceName() == "server-1") { // Start task to periodically check if someone's trying to upgrade (new Task(() => CheckForUpgradeAsync())).Start(); @@ -2666,35 +2678,61 @@ private void FinishResharding() private void EstablishInputConnections(ConcurrentDictionary states) { - foreach (var state in states.Values) + Connect(ServiceName(), AmbrosiaDataOutputsName, ServiceName(), AmbrosiaDataInputsName); + Connect(ServiceName(), AmbrosiaControlOutputsName, ServiceName(), AmbrosiaControlInputsName); + Console.WriteLine("Establishing connections for " + ServiceName()); + var connectionResult1 = Connect(ServiceName(), AmbrosiaDataOutputsName, "client2-1", AmbrosiaDataInputsName); + var connectionResult2 = Connect(ServiceName(), AmbrosiaControlOutputsName, "client2-1", AmbrosiaControlInputsName); + var connectionResult3 = Connect("client2-1", AmbrosiaDataOutputsName, ServiceName(), AmbrosiaDataInputsName); + var connectionResult4 = Connect("client2-1", AmbrosiaControlOutputsName, ServiceName(), AmbrosiaControlInputsName); + if ((connectionResult1 != CRAErrorCode.Success) || (connectionResult2 != CRAErrorCode.Success) || + (connectionResult3 != CRAErrorCode.Success) || (connectionResult4 != CRAErrorCode.Success)) { - foreach (var kv in state.Inputs) - { - var destination = kv.Key; - if (destination == "") - { - destination = ServiceName(); - } - List results = new List(); - results.Add(Connect(ServiceName(), AmbrosiaDataOutputsName, destination, AmbrosiaDataInputsName)); - results.Add(Connect(ServiceName(), AmbrosiaControlOutputsName, destination, AmbrosiaControlInputsName)); - if (destination != ServiceName()) - { - results.Add(Connect(destination, AmbrosiaDataOutputsName, ServiceName(), AmbrosiaDataInputsName)); - results.Add(Connect(destination, AmbrosiaControlOutputsName, ServiceName(), AmbrosiaControlInputsName)); - } + Console.WriteLine("Error attaching " + ServiceName() + " to " + "client2-1"); + // BUGBUG in tests. Should exit here. Fix tests then delete above line and replace with this OnError(0, "Error attaching " + _serviceName + " to " + destination); + } + connectionResult1 = Connect(ServiceName(), AmbrosiaDataOutputsName, "client1-1", AmbrosiaDataInputsName); + connectionResult2 = Connect(ServiceName(), AmbrosiaControlOutputsName, "client1-1", AmbrosiaControlInputsName); + connectionResult3 = Connect("client1-1", AmbrosiaDataOutputsName, ServiceName(), AmbrosiaDataInputsName); + connectionResult4 = Connect("client1-1", AmbrosiaControlOutputsName, ServiceName(), AmbrosiaControlInputsName); + if ((connectionResult1 != CRAErrorCode.Success) || (connectionResult2 != CRAErrorCode.Success) || + (connectionResult3 != CRAErrorCode.Success) || (connectionResult4 != CRAErrorCode.Success)) + { - foreach (var result in results) - { - if (result != CRAErrorCode.Success) - { - Console.WriteLine("EstablishInputConnections: Error connecting to " + destination); - break; - } - } - } + Console.WriteLine("Error attaching " + ServiceName() + " to " + "client1-1"); + // BUGBUG in tests. Should exit here. Fix tests then delete above line and replace with this OnError(0, "Error attaching " + _serviceName + " to " + destination); } + /* foreach (var state in states.Values) + { + foreach (var kv in state.Inputs) + { + var destination = kv.Key; + if (destination == "") + { + destination = ServiceName(); + } + + List results = new List(); + Console.WriteLine("Establishing connection between {0} and {1}", ServiceName(), destination); + results.Add(Connect(ServiceName(), AmbrosiaDataOutputsName, destination, AmbrosiaDataInputsName)); + results.Add(Connect(ServiceName(), AmbrosiaControlOutputsName, destination, AmbrosiaControlInputsName)); + if (destination != ServiceName()) + { + results.Add(Connect(destination, AmbrosiaDataOutputsName, ServiceName(), AmbrosiaDataInputsName)); + results.Add(Connect(destination, AmbrosiaControlOutputsName, ServiceName(), AmbrosiaControlInputsName)); + } + + foreach (var result in results) + { + if (result != CRAErrorCode.Success) + { + Console.WriteLine("EstablishInputConnections: Error connecting to " + destination); + break; + } + } + } + }*/ } private void RecoverFromShards(long checkpointToLoad = -1, bool testUpgrade = false) @@ -2706,6 +2744,8 @@ private void RecoverFromShards(long checkpointToLoad = -1, bool testUpgrade = fa _inputs = new ConcurrentDictionary(); _outputs = new ConcurrentDictionary(); + + EstablishInputConnections(_parentStates); _committer = new Committer(_localServiceSendToStream, _persistLogs, this); for (int i = 0; i < _oldShards.Length; i++) @@ -2728,7 +2768,6 @@ private void RecoverFromShards(long checkpointToLoad = -1, bool testUpgrade = fa } AddParentStates(_parentStates.Values.ToArray()); - EstablishInputConnections(_parentStates); // Wait for replay for all shards to occur for (int i = 0; i < threads.Length; i++) @@ -3114,6 +3153,7 @@ internal async Task DetectBecomingPrimaryAsync(MachineState state) await state.Committer.WakeupAsync(); state.MyRole = AARole.Primary; // this will stop and break the loop in the function replayInput_Sec() + Console.WriteLine("Primary {0}", (DateTime.Now - new DateTime(1970, 1, 1)).TotalMilliseconds); Console.WriteLine("\n\nNOW I'm Primary\n\n"); // if we are an upgrader : Time to release the kill file lock and cleanup. Note that since we have the log lock // everyone is prevented from promotion until we succeed or fail. @@ -3198,10 +3238,11 @@ private async Task ReplayAsync(LogReader replayStream, MachineState state) } replayStream.Read(tempBuf2, 0, inputNameSize); var inputName = Encoding.UTF8.GetString(tempBuf2, 0, inputNameSize); + var newLongPair = new LongPair(); newLongPair.First = replayStream.ReadLongFixed(); newLongPair.Second = replayStream.ReadLongFixed(); - committedInputDict[inputName] = newLongPair; + committedInputDict[inputName] = newLongPair; } // Read changes in trim to perform and reflect in _outputs watermarksToRead = replayStream.ReadInt(); @@ -3217,6 +3258,7 @@ private async Task ReplayAsync(LogReader replayStream, MachineState state) var inputName = Encoding.UTF8.GetString(tempBuf2, 0, inputNameSize); long seqNo = replayStream.ReadLongFixed(); trimDict[inputName] = seqNo; + } } catch @@ -3355,6 +3397,7 @@ private async Task ReplayAsync(LogReader replayStream, MachineState state) { if (!state.Outputs.TryGetValue(kv.Key, out outputConnectionRecord)) { + Console.WriteLine("OUTPUT " + ServiceName() + " " + kv.Key); outputConnectionRecord = new OutputConnectionRecord(this); state.Outputs[kv.Key] = outputConnectionRecord; } @@ -3491,6 +3534,13 @@ private string DestinationShard(string destination, long shardID = -1) { return destination; } + + Console.WriteLine("DESTINATION " + destination); + if (_serviceNames.ContainsKey(destination)) + { + return _serviceNames[destination]; + } + if (shardID == -1 && _sharded) { shardID = _shardLocator(BitConverter.ToInt32(Encoding.UTF8.GetBytes(destination), 0)); @@ -3507,6 +3557,7 @@ private void ProcessSyncLocalMessage(ref FlexReadBuffer localServiceBuffer, Flex switch (localServiceBuffer.Buffer[sizeBytes]) { case takeCheckpointByte: + // Handle take checkpoint messages - This is here for testing createCheckpointTask = new Task(new Action(MoveServiceToNextLogFileSimple)); createCheckpointTask.Start(); @@ -3536,7 +3587,7 @@ private void ProcessSyncLocalMessage(ref FlexReadBuffer localServiceBuffer, Flex } else { - Console.WriteLine("Attaching to {0}", destination); + Console.WriteLine("Attaching to {0} from {1}", destination, ServiceName()); var connectionResult1 = Connect(ServiceName(), AmbrosiaDataOutputsName, destination, AmbrosiaDataInputsName); var connectionResult2 = Connect(ServiceName(), AmbrosiaControlOutputsName, destination, AmbrosiaControlInputsName); var connectionResult3 = Connect(destination, AmbrosiaDataOutputsName, ServiceName(), AmbrosiaDataInputsName); @@ -3553,6 +3604,7 @@ private void ProcessSyncLocalMessage(ref FlexReadBuffer localServiceBuffer, Flex break; case RPCBatchByte: + var restOfBatchOffset = sizeBytes + 1; var memStream = new MemoryStream(localServiceBuffer.Buffer, restOfBatchOffset, localServiceBuffer.Length - restOfBatchOffset); var numRPCs = memStream.ReadInt(); @@ -3576,6 +3628,7 @@ private void ProcessSyncLocalMessage(ref FlexReadBuffer localServiceBuffer, Flex break; case RPCByte: + ProcessRPC(localServiceBuffer); // Now process any pending RPC requests from the local service before going async again break; @@ -3733,6 +3786,9 @@ private async Task SyncOutputConnectionAsync(ConcurrentD { destString = ""; } + InputConnectionRecord inputConnectionRecord; + + lock (outputs) { if (!outputs.TryGetValue(destString, out outputConnectionRecord)) @@ -3915,6 +3971,13 @@ private async Task ToDataStreamAsync(Stream writeToStream, { destString = ""; } + if (destString == "server-3") + { + _serviceNames["server-1"] = "server-3"; + _serviceNames["server-2"] = "server-3"; + _lastShuffleDest = null; + } else + if (_sharded) { Serializer.SerializeAncestorMessage(writeToStream, ancestorByte, _ancestors[ServiceName()]); @@ -4037,6 +4100,12 @@ private async Task ToControlStreamAsync(Stream writeToStream, { destString = ""; } + if (destString == "server-3") + { + _serviceNames["server-1"] = "server-3"; + _serviceNames["server-2"] = "server-3"; + } + OutputConnectionRecord outputConnectionRecord = GetOutputConnectionRecord(destString); // Process remote trim message var inputFlexBuffer = new FlexReadBuffer(); @@ -4149,6 +4218,11 @@ private async Task FromDataStreamAsync(Stream readFromStream, inputConnectionRecord.ShardID = ParseServiceName(sourceString).Item2; _inputs[sourceString] = inputConnectionRecord; Console.WriteLine("Adding input:{0}", sourceString); + if (sourceString == "server-3") + { + _serviceNames["server-1"] = "server-3"; + _serviceNames["server-2"] = "server-3"; + } } else { @@ -4227,6 +4301,7 @@ private async Task InputDataListenerAsync(InputConnectionRecord inputRecord, while (true) { await FlexReadBuffer.DeserializeAsync(inputRecord.DataConnectionStream, inputFlexBuffer, ct); + Console.WriteLine("Receive data from " + inputName); await ProcessInputMessageAsync(inputRecord, inputName, inputFlexBuffer); } } @@ -4429,6 +4504,10 @@ private void TrimGlobalOutputTracker() public void TrimOutput(string key, long lastProcessedID, long lastProcessedReplayableID) { +<<<<<<< Updated upstream +======= + Console.WriteLine("*X* Trimming {0} up to ({1}, {2})", key, lastProcessedID, lastProcessedReplayableID); +>>>>>>> Stashed changes OutputConnectionRecord outputConnectionRecord; if (!_outputs.TryGetValue(key, out outputConnectionRecord)) { @@ -4502,16 +4581,7 @@ public long KeyHashToShard(long hash) return -1; } - if (_serviceName.Contains("client")) - { - foreach (var input in _inputs.Keys) - { - if (input.Contains("server-2")) - { - return 2; - } - } - } + new Exception(""); return 1; } @@ -4526,13 +4596,16 @@ public override void Initialize(object param) p = (AmbrosiaRuntimeParams)xmlSerializer.Deserialize(textReader); } - bool runningRepro = false; + _runningRepro = false; bool sharded = p.shardID > 0; _shardID = p.shardID; _shardLocator = KeyHashToShard; _oldShards = p.oldShards; _newShards = p.newShards; + _serviceNames["server-1"] = "server-1"; + _serviceNames["server-2"] = "server-2"; + Initialize( p.serviceReceiveFromPort, p.serviceSendToPort, @@ -4546,7 +4619,6 @@ public override void Initialize(object param) p.storageConnectionString, p.currentVersion, p.upgradeToVersion, - runningRepro, sharded ); } @@ -4822,7 +4894,15 @@ static void Main(string[] args) serializedParams = textWriter.ToString(); } +<<<<<<< Updated upstream InstantiateVertex(client, replicaName, shardName, param.AmbrosiaBinariesLocation, serializedParams, _shardID > 0); +======= + if (client.InstantiateVertex(replicaName, shardName, param.AmbrosiaBinariesLocation, serializedParams) != CRAErrorCode.Success) + { + throw new Exception(); + } + Console.WriteLine("Creating endpoints for {0}", shardName); +>>>>>>> Stashed changes client.AddEndpoint(shardName, AmbrosiaRuntime.AmbrosiaDataInputsName, true, true); client.AddEndpoint(shardName, AmbrosiaRuntime.AmbrosiaDataOutputsName, false, true); diff --git a/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs b/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs index 4e277217..9e5cee79 100644 --- a/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs +++ b/AmbrosiaTest/AmbrosiaTest/Shard_UnitTest.cs @@ -191,7 +191,7 @@ public void Shard_UnitTest_SingleReshardEndtoEnd_Test() int clientJobProcessID = MyUtils.StartPerfClientJob("1001", "1000", clientJobName, serverName, "1024", "1", logOutputFileName_ClientJob); // Give it a few seconds to start - Thread.Sleep(2000); + Thread.Sleep(1000); // First Server Call string logOutputFileName_Server1 = testName + "_Server1.log"; diff --git a/AmbrosiaTest/AmbrosiaTest/app.config b/AmbrosiaTest/AmbrosiaTest/app.config index 96bcda5d..a3b83fb4 100644 --- a/AmbrosiaTest/AmbrosiaTest/app.config +++ b/AmbrosiaTest/AmbrosiaTest/app.config @@ -11,7 +11,7 @@ - + diff --git a/Samples/HelloWorld/Client1/Program.cs b/Samples/HelloWorld/Client1/Program.cs index 7e7da210..9455a1fc 100644 --- a/Samples/HelloWorld/Client1/Program.cs +++ b/Samples/HelloWorld/Client1/Program.cs @@ -5,6 +5,7 @@ using System; using System.Runtime.Serialization; using System.Threading.Tasks; +using System.Threading; namespace Client1 { @@ -40,8 +41,15 @@ protected override async Task OnFirstStart() { _server = GetProxy(_serverName); + for (int i = 0; i < 1000; i++) + { + Console.WriteLine("Client 1: Hello World {0}", 2*i); + _server.ReceiveMessageFork("Client 1: Hello World " + (2*i).ToString()); + Thread.Sleep(1000); + } - _server.ReceiveMessageFork("\n!! Client: Hello World 1!"); + + /*_server.ReceiveMessageFork("\n!! Client: Hello World 1!"); using (ConsoleColorScope.SetForeground(ConsoleColor.Yellow)) { @@ -56,7 +64,7 @@ protected override async Task OnFirstStart() using (ConsoleColorScope.SetForeground(ConsoleColor.Yellow)) { Console.WriteLine("\n!! Client: Press enter to shutdown."); - } + }*/ Console.ReadLine(); Program.finishedTokenQ.Enqueue(0); @@ -73,8 +81,8 @@ static void Main(string[] args) int receivePort = 1001; int sendPort = 1000; - string clientInstanceName = "client"; - string serverInstanceName = "server"; + string clientInstanceName = "client1"; + string serverInstanceName = "server-1"; if (args.Length >= 1) { diff --git a/Samples/HelloWorld/Client2/Program.cs b/Samples/HelloWorld/Client2/Program.cs index 949f2970..43fb76a2 100644 --- a/Samples/HelloWorld/Client2/Program.cs +++ b/Samples/HelloWorld/Client2/Program.cs @@ -34,12 +34,12 @@ void InputLoop() } } - protected override void BecomingPrimary() + /* protected override void BecomingPrimary() { Console.WriteLine("Finished initializing state/recovering"); Thread timerThread = new Thread(InputLoop); timerThread.Start(); - } + }*/ public async Task ReceiveKeyboardInputAsync(string input) { @@ -50,6 +50,34 @@ public async Task ReceiveKeyboardInputAsync(string input) protected override async Task OnFirstStart() { _server = GetProxy(_serverName); + + for (int i = 0; i < 1000; i++) + { + Console.WriteLine("Client 2: Hello World {0}", 2 * i + 1); + _server.ReceiveMessageFork("Client 2: Hello World " + (2 * i + 1).ToString()); + Thread.Sleep(1000); + } + + + /*_server.ReceiveMessageFork("\n!! Client: Hello World 1!"); + + using (ConsoleColorScope.SetForeground(ConsoleColor.Yellow)) + { + Console.WriteLine("\n!! Client: Sent message 1."); + Console.WriteLine("\n!! Client: Press enter to continue (will send 2&3)"); + } + + Console.ReadLine(); + _server.ReceiveMessageFork("\n!! Client: Hello World 2!"); + _server.ReceiveMessageFork("\n!! Client: Hello World 3!"); + + using (ConsoleColorScope.SetForeground(ConsoleColor.Yellow)) + { + Console.WriteLine("\n!! Client: Press enter to shutdown."); + }*/ + + Console.ReadLine(); + Program.finishedTokenQ.Enqueue(0); return true; } } @@ -62,10 +90,10 @@ static void Main(string[] args) { finishedTokenQ = new AsyncQueue(); - int receivePort = 1001; - int sendPort = 1000; - string clientInstanceName = "client"; - string serverInstanceName = "server"; + int receivePort = 3001; + int sendPort = 3000; + string clientInstanceName = "client2"; + string serverInstanceName = "server-2"; if (args.Length >= 1) { diff --git a/Samples/HelloWorld/Server/Program.cs b/Samples/HelloWorld/Server/Program.cs index 5478c64e..4b564d61 100644 --- a/Samples/HelloWorld/Server/Program.cs +++ b/Samples/HelloWorld/Server/Program.cs @@ -1,5 +1,5 @@ using Ambrosia; -using Server; +using Client1; using System; using System.Runtime.Serialization; using System.Threading; @@ -29,6 +29,9 @@ sealed class Server : Immortal, IServer [DataMember] int _messagesReceived = 0; + private IClient1Proxy _client1; + private IClient2Proxy _client2; + public Server() { } @@ -46,6 +49,8 @@ public async Task ReceiveMessageAsync(string message) protected override async Task OnFirstStart() { + _client1 = GetProxy("client1"); + _client2 = GetProxy("client2"); return true; } } @@ -64,6 +69,9 @@ static void Main(string[] args) { sendPort = int.Parse(args[1]); } + + + if (args.Length == 3) { serviceName = args[2];