diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerCV.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerCV.cs new file mode 100644 index 0000000000..f6e39cc88a --- /dev/null +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerCV.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NBitcoin; +using Stratis.Bitcoin.Builder.Feature; +using Stratis.Bitcoin.Consensus; +using Stratis.Bitcoin.Controllers.Models; +using Stratis.Bitcoin.Features.BlockStore.Models; +using Stratis.Bitcoin.Features.Consensus.CoinViews; +using Stratis.Bitcoin.Interfaces; + +namespace Stratis.Bitcoin.Features.BlockStore.AddressIndexing +{ + public class AddressIndexerCV : IAddressIndexer + { + private readonly Network network; + private readonly ICoinView coinView; + private readonly ChainIndexer chainIndexer; + private readonly IScriptAddressReader scriptAddressReader; + + public ChainedHeader IndexerTip => GetTip(); + + public IFullNodeFeature InitializingFeature { set; private get; } + + public AddressIndexerCV(Network network, ChainIndexer chainIndexer, IScriptAddressReader scriptAddressReader, ICoinView coinView) + { + this.network = network; + this.coinView = coinView; + this.chainIndexer = chainIndexer; + this.scriptAddressReader = scriptAddressReader; + } + + private ChainedHeader GetTip() + { + this.coinView.Sync(this.chainIndexer); + + return this.chainIndexer[this.coinView.GetTipHash().Hash]; + } + + public void Initialize() + { + } + + private TxDestination AddressToDestination(string address) + { + var bitcoinAddress = BitcoinAddress.Create(address, this.network); + return this.scriptAddressReader.GetDestinationFromScriptPubKey(this.network, bitcoinAddress.ScriptPubKey).Single(); + } + + public AddressBalancesResult GetAddressBalances(string[] addresses, int minConfirmations = 0) + { + return new AddressBalancesResult() + { + Balances = addresses + .Select(address => (address, destination: AddressToDestination(address))) + .Select(t => new AddressBalanceResult() + { + Address = t.address, + Balance = (t.destination == null) ? 0 : new Money(this.coinView.GetBalance(t.destination).First().satoshis), + + }).ToList() + }; + } + + public LastBalanceDecreaseTransactionModel GetLastBalanceDecreaseTransaction(string address) + { + throw new NotImplementedException(); + } + + private IEnumerable ToDiff(List addressBalanceChanges) + { + for (int i = addressBalanceChanges.Count - 1; i > 0; i--) + { + yield return new AddressBalanceChange() { + BalanceChangedHeight = addressBalanceChanges[i - 1].BalanceChangedHeight, + Deposited = addressBalanceChanges[i - 1].Satoshi < addressBalanceChanges[i].Satoshi, + Satoshi = Math.Abs(addressBalanceChanges[i - 1].Satoshi - addressBalanceChanges[i].Satoshi) + }; + } + } + + /// + public VerboseAddressBalancesResult GetAddressIndexerState(string[] addresses) + { + // If the containing feature is not initialized then wait a bit. + this.InitializingFeature?.WaitInitialized(); + + return new VerboseAddressBalancesResult(this.IndexerTip.Height) + { + BalancesData = addresses + .Select(address => (address, destination: AddressToDestination(address))) + .Select(t => new AddressIndexerData() + { + Address = t.address, + BalanceChanges = (t.destination == null) ? new List() : ToDiff(this.coinView.GetBalance(t.destination).Select(b => new AddressBalanceChange() + { + BalanceChangedHeight = (int)b.height, + Deposited = b.satoshis >= 0, + Satoshi = Math.Abs(b.satoshis) + }).ToList()).ToList() + }).ToList() + }; + } + + public void Dispose() + { + } + } +} diff --git a/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs b/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs index 10d2f550eb..00fee016bb 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs @@ -199,7 +199,7 @@ public static IFullNodeBuilder UseBlockStore(this IFullNodeBuilder fullNodeBuild services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); }); }); diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs index 8f882403a4..8767ac6cfb 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs @@ -43,7 +43,7 @@ public CoinviewTests() this.nodeStats = new NodeStats(this.dateTimeProvider, NodeSettings.Default(this.network), new Mock().Object); this.coindb = new DBreezeCoindb(this.network, this.dataFolder, this.dateTimeProvider, this.loggerFactory, this.nodeStats, new DBreezeSerializer(this.network.Consensus.ConsensusFactory)); - this.coindb.Initialize(new ChainedHeader(this.network.GetGenesis().Header, this.network.GenesisHash, 0)); + this.coindb.Initialize(false); this.chainIndexer = new ChainIndexer(this.network); this.stakeChainStore = new StakeChainStore(this.network, this.chainIndexer, (IStakedb)this.coindb, this.loggerFactory); diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs index e77ad63bb0..c611a1ae8a 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs @@ -5,8 +5,10 @@ using System.Threading; using Microsoft.Extensions.Logging; using NBitcoin; +using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Configuration.Settings; using Stratis.Bitcoin.Consensus; +using Stratis.Bitcoin.Consensus.Rules; using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders; using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules; using Stratis.Bitcoin.Interfaces; @@ -114,6 +116,10 @@ public long GetScriptSize /// All access to this object has to be protected by . private readonly Dictionary cachedUtxoItems; + /// Tracks pending balance updates for dirty cache entries. + /// All access to this object has to be protected by . + private readonly Dictionary> cacheBalancesByDestination; + /// Number of items in the cache. /// The getter violates the lock contract on , but the lock here is unnecessary as the is marked as readonly. private int cacheCount => this.cachedUtxoItems.Count; @@ -127,35 +133,38 @@ public long GetScriptSize private long rewindDataSizeBytes; private DateTime lastCacheFlushTime; private readonly Network network; - private readonly ICheckpoints checkpoints; private readonly IDateTimeProvider dateTimeProvider; private readonly IBlockStore blockStore; private readonly CancellationTokenSource cancellationToken; private readonly ConsensusSettings consensusSettings; + private readonly bool addressIndexingEnabled; private CachePerformanceSnapshot latestPerformanceSnapShot; + private IScriptAddressReader scriptAddressReader; private readonly Random random; - public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, - StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IBlockStore blockStore = null, INodeLifetime nodeLifetime = null) + public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, + StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IScriptAddressReader scriptAddressReader = null, IBlockStore blockStore = null, INodeLifetime nodeLifetime = null, NodeSettings nodeSettings = null) { Guard.NotNull(coindb, nameof(CachedCoinView.coindb)); this.coindb = coindb; this.logger = loggerFactory.CreateLogger(this.GetType().FullName); this.network = network; - this.checkpoints = checkpoints; this.dateTimeProvider = dateTimeProvider; this.consensusSettings = consensusSettings; this.stakeChainStore = stakeChainStore; this.rewindDataIndexCache = rewindDataIndexCache; this.blockStore = blockStore; - this.cancellationToken = (nodeLifetime == null) ? new CancellationTokenSource() : CancellationTokenSource.CreateLinkedTokenSource(nodeLifetime.ApplicationStopping); + this.cancellationToken = (nodeLifetime == null) ? new CancellationTokenSource(): CancellationTokenSource.CreateLinkedTokenSource(nodeLifetime.ApplicationStopping); this.lockobj = new object(); this.cachedUtxoItems = new Dictionary(); + this.cacheBalancesByDestination = new Dictionary>(); this.performanceCounter = new CachePerformanceCounter(this.dateTimeProvider); this.lastCacheFlushTime = this.dateTimeProvider.GetUtcNow(); this.cachedRewindData = new Dictionary(); + this.scriptAddressReader = scriptAddressReader; + this.addressIndexingEnabled = nodeSettings?.ConfigReader.GetOrDefault("addressindex", false) ?? false; this.random = new Random(); this.MaxCacheSizeBytes = consensusSettings.MaxCoindbCacheInMB * 1024 * 1024; @@ -200,7 +209,7 @@ public void Sync(ChainIndexer chainIndexer) public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine) { - this.coindb.Initialize(chainTip); + this.coindb.Initialize(this.addressIndexingEnabled); Sync(chainIndexer); @@ -469,10 +478,11 @@ public void Flush(bool force = true) this.logger.LogDebug("Flushing {0} items.", modify.Count); - this.coindb.SaveChanges(modify, this.innerBlockHash, this.blockHash, this.cachedRewindData.Select(c => c.Value).ToList()); + this.coindb.SaveChanges(modify, this.cacheBalancesByDestination, this.innerBlockHash, this.blockHash, this.cachedRewindData.Select(c => c.Value).ToList()); // All the cached utxos are now on disk so we can clear the cached entry list. this.cachedUtxoItems.Clear(); + this.cacheBalancesByDestination.Clear(); this.cacheSizeBytes = 0; this.cachedRewindData.Clear(); @@ -564,6 +574,10 @@ public void SaveChanges(IList outputs, HashHeightPair oldBlockHas { // DELETE COINS + // Record the UTXO as having been spent at this height. + if (cacheItem.Coins != null) + this.RecordBalanceChange(cacheItem.Coins.TxOut.ScriptPubKey, -cacheItem.Coins.TxOut.Value, (uint)nextBlockHash.Height); + // In cases of an output spent in the same block // it wont exist in cash or in disk so its safe to remove it if (cacheItem.Coins == null) @@ -612,6 +626,9 @@ public void SaveChanges(IList outputs, HashHeightPair oldBlockHas { // ADD COINS + // Update the balance. + this.RecordBalanceChange(output.Coins.TxOut.ScriptPubKey, output.Coins.TxOut.Value, output.Coins.Height); + if (cacheItem.Coins != null) { // Allow overrides. @@ -693,6 +710,7 @@ public HashHeightPair Rewind(HashHeightPair target = null) // All the cached utxos are now on disk so we can clear the cached entry list. this.cachedUtxoItems.Clear(); + this.cacheBalancesByDestination.Clear(); this.cacheSizeBytes = 0; this.dirtyCacheCount = 0; @@ -749,5 +767,62 @@ private void AddBenchStats(StringBuilder log) this.latestPerformanceSnapShot = snapShot; } + + private void RecordBalanceChange(Script scriptPubKey, long satoshis, uint height) + { + if (!this.coindb.BalanceIndexingEnabled || scriptPubKey.Length == 0 || satoshis == 0) + return; + + foreach (TxDestination txDestination in this.scriptAddressReader.GetDestinationFromScriptPubKey(this.network, scriptPubKey)) + { + if (!this.cacheBalancesByDestination.TryGetValue(txDestination, out Dictionary value)) + { + value = new Dictionary(); + this.cacheBalancesByDestination[txDestination] = value; + } + + if (!value.TryGetValue(height, out long balance)) + balance = 0; + + balance += satoshis; + + value[height] = balance; + } + } + + public IEnumerable<(uint, long)> GetBalance(TxDestination txDestination) + { + IEnumerable<(uint, long)> CachedBalances() + { + if (this.cacheBalancesByDestination.TryGetValue(txDestination, out Dictionary itemsByHeight)) + { + long balance = 0; + + foreach (uint height in itemsByHeight.Keys.OrderBy(k => k)) + { + balance += itemsByHeight[height]; + yield return (height, balance); + } + } + } + + bool first = true; + foreach ((uint height, long satoshis) in this.coindb.GetBalance(txDestination)) + { + if (first) + { + first = false; + + foreach ((uint height2, long satoshis2) in CachedBalances().Reverse()) + yield return (height2, satoshis2 + satoshis); + } + + yield return (height, satoshis); + } + + if (first) + foreach ((uint height2, long satoshis2) in CachedBalances().Reverse()) + yield return (height2, satoshis2); + } } } diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs index eca612cdb9..7b68b84274 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs @@ -84,5 +84,15 @@ public interface ICoinView /// /// The height of the block. RewindData GetRewindData(int height); + + /// + /// Returns a combination of (height, satoshis) values with the cumulative balance up to the corresponding height. + /// + /// The destination value derived from the address being queried. + /// A combination of (height, satoshis) values with the cumulative balance up to the corresponding height. + /// Balance updates (even when nett 0) are delivered for every height at which transactions for the address + /// had been recorded and as such the returned heights can be used in conjunction with the block store to discover + /// all related transactions. + IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination); } } diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index be0cd8603a..2b48ddb927 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -6,7 +6,9 @@ using NBitcoin; using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Configuration.Logging; +using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Database; +using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.Consensus.CoinViews @@ -24,6 +26,11 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews private static readonly byte blockTable = 2; private static readonly byte rewindTable = 3; private static readonly byte stakeTable = 4; + private static readonly byte balanceTable = 5; + private static readonly byte balanceAdjustmentTable = 6; + + /// Database key under which the block hash of the coin view's last indexed tip is stored. + private static readonly byte[] blockIndexedHashKey = new byte[1]; private readonly string dataFolder; @@ -33,15 +40,19 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews /// Specification of the network the node runs on - regtest/testnet/mainnet. private readonly Network network; + public bool BalanceIndexingEnabled { get; private set; } + /// Hash of the block which is currently the tip of the coinview. private HashHeightPair persistedCoinviewTip; + private readonly IScriptAddressReader scriptAddressReader; + /// Performance counter to measure performance of the database insert and query operations. private readonly BackendPerformanceCounter performanceCounter; private BackendPerformanceSnapshot latestPerformanceSnapShot; - /// Access to dBreeze database. + /// Access to database. private IDb coinDb; private readonly DBreezeSerializer dBreezeSerializer; @@ -49,13 +60,13 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews private const int MaxRewindBatchSize = 10000; public Coindb(Network network, DataFolder dataFolder, IDateTimeProvider dateTimeProvider, - INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) - : this(network, dataFolder.CoindbPath, dateTimeProvider, nodeStats, dBreezeSerializer) + INodeStats nodeStats, DBreezeSerializer dBreezeSerializer, IScriptAddressReader scriptAddressReader) + : this(network, dataFolder.CoindbPath, dateTimeProvider, nodeStats, dBreezeSerializer, scriptAddressReader) { } - public Coindb(Network network, string dataFolder, IDateTimeProvider dateTimeProvider, - INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) + public Coindb(Network network, string dataFolder, IDateTimeProvider dateTimeProvider, + INodeStats nodeStats, DBreezeSerializer dBreezeSerializer, IScriptAddressReader scriptAddressReader) { Guard.NotNull(network, nameof(network)); Guard.NotEmpty(dataFolder, nameof(dataFolder)); @@ -65,79 +76,29 @@ public Coindb(Network network, string dataFolder, IDateTimeProvider dateTimeProv this.logger = LogManager.GetCurrentClassLogger(); this.network = network; this.performanceCounter = new BackendPerformanceCounter(dateTimeProvider); + this.scriptAddressReader = scriptAddressReader; if (nodeStats.DisplayBenchStats) nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 400); } - public void Initialize(ChainedHeader chainTip) + /// + public void Initialize(bool balanceIndexingEnabled) { // Open a connection to a new DB and create if not found this.coinDb = new T(); this.coinDb.Open(this.dataFolder); - // Check if key bytes are in the wrong endian order. - HashHeightPair current = this.GetTipHash(); - - if (current != null) - { - byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(current.Height)); - // Fix the table if required. - if (row != null) - { - // To be sure, check the next height too. - byte[] row2 = (current.Height > 1) ? this.coinDb.Get(rewindTable, BitConverter.GetBytes(current.Height - 1)) : new byte[] { }; - if (row2 != null) - { - this.logger.LogInformation("Fixing the coin db."); - - var rows = new Dictionary(); - - using (var iterator = this.coinDb.GetIterator(rewindTable)) - { - iterator.Seek(new byte[0]); - - while (iterator.IsValid()) - { - byte[] key = iterator.Key(); - - if (key.Length != 4) - break; + this.BalanceIndexingEnabled = balanceIndexingEnabled; - int height = BitConverter.ToInt32(key); - - rows[height] = iterator.Value(); - - iterator.Next(); - } - } - - using (var batch = this.coinDb.GetWriteBatch()) - { - foreach (int height in rows.Keys.OrderBy(k => k)) - { - batch.Delete(rewindTable, BitConverter.GetBytes(height)); - } - - foreach (int height in rows.Keys.OrderBy(k => k)) - { - batch.Put(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray(), rows[height]); - } - - batch.Write(); - } - } - } - } - - EnsureCoinDatabaseIntegrity(chainTip); - - Block genesis = this.network.GetGenesis(); + EnsureCoinDatabaseIntegrity(); if (this.GetTipHash() == null) { using (var batch = this.coinDb.GetWriteBatch()) { + Block genesis = this.network.GetGenesis(); + this.SetBlockHash(batch, new HashHeightPair(genesis.GetHash(), 0)); batch.Write(); } @@ -146,55 +107,7 @@ public void Initialize(ChainedHeader chainTip) this.logger.LogInformation("Coinview initialized with tip '{0}'.", this.persistedCoinviewTip); } - private void EnsureCoinDatabaseIntegrity(ChainedHeader chainTip) - { - this.logger.LogInformation("Checking coin database integrity..."); - - var heightToCheck = chainTip.Height; - - // Find the height up to where rewind data is stored above chain tip. - do - { - heightToCheck += 1; - - byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(heightToCheck).Reverse().ToArray()); - if (row == null) - break; - - } while (true); - - for (int height = heightToCheck - 1; height > chainTip.Height;) - { - this.logger.LogInformation($"Fixing coin database, deleting rewind data at height {height} above tip '{chainTip}'."); - - // Do a batch of rewinding. - height = RewindInternal(height, new HashHeightPair(chainTip)).Height; - } - - this.logger.LogInformation("Coin database integrity good."); - } - - private void SetBlockHash(IDbBatch batch, HashHeightPair nextBlockHash) - { - this.persistedCoinviewTip = nextBlockHash; - batch.Put(blockTable, blockHashKey, nextBlockHash.ToBytes()); - } - - public HashHeightPair GetTipHash() - { - if (this.persistedCoinviewTip == null) - { - var row = this.coinDb.Get(blockTable, blockHashKey); - if (row != null) - { - this.persistedCoinviewTip = new HashHeightPair(); - this.persistedCoinviewTip.FromBytes(row); - } - } - - return this.persistedCoinviewTip; - } - + /// public FetchCoinsResponse FetchCoins(OutPoint[] utxos) { FetchCoinsResponse res = new FetchCoinsResponse(); @@ -217,12 +130,15 @@ public FetchCoinsResponse FetchCoins(OutPoint[] utxos) return res; } - public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) + /// + public void SaveChanges(IList unspentOutputs, Dictionary> balanceUpdates, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) { int insertedEntities = 0; - using (var batch = this.coinDb.GetWriteBatch()) + using (var batch = this.coinDb.GetReadWriteBatch()) { + this.AdjustBalance(batch, balanceUpdates); + using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) { HashHeightPair current = this.GetTipHash(); @@ -280,105 +196,271 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB } /// - public int GetMinRewindHeight() + public HashHeightPair Rewind(HashHeightPair target) + { + HashHeightPair current = this.GetTipHash(); + return RewindInternal(current.Height, target); + } + + /// + public RewindData GetRewindData(int height) { - // Find the first row with a rewind table key prefix. - using (var iterator = this.coinDb.GetIterator(rewindTable)) + byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); + return row != null ? this.dBreezeSerializer.Deserialize(row) : null; + } + + /// + public void PutStake(IEnumerable stakeEntries) + { + using (var batch = this.coinDb.GetWriteBatch()) { - iterator.Seek(new byte[0]); - if (!iterator.IsValid()) - return -1; + foreach (StakeItem stakeEntry in stakeEntries) + { + if (!stakeEntry.InStore) + { + batch.Put(stakeTable, stakeEntry.BlockId.ToBytes(false), this.dBreezeSerializer.Serialize(stakeEntry.BlockStake)); + stakeEntry.InStore = true; + } + } - byte[] key = iterator.Key(); + batch.Write(); + } + } - if (key.Length != 4) - return -1; + /// + public void GetStake(IEnumerable blocklist) + { + foreach (StakeItem blockStake in blocklist) + { + this.logger.LogTrace("Loading POS block hash '{0}' from the database.", blockStake.BlockId); + byte[] stakeRow = this.coinDb.Get(stakeTable, blockStake.BlockId.ToBytes(false)); - return BitConverter.ToInt32(key.SafeSubarray(0, 4).Reverse().ToArray()); + if (stakeRow != null) + { + blockStake.BlockStake = this.dBreezeSerializer.Deserialize(stakeRow); + blockStake.InStore = true; + } } } - public HashHeightPair Rewind(HashHeightPair target) + /// + public HashHeightPair GetTipHash() { - HashHeightPair current = this.GetTipHash(); - return RewindInternal(current.Height, target); + if (this.persistedCoinviewTip == null) + { + var row = this.coinDb.Get(blockTable, blockHashKey); + if (row != null) + { + this.persistedCoinviewTip = new HashHeightPair(); + this.persistedCoinviewTip.FromBytes(row); + } + } + + return this.persistedCoinviewTip; + } + + private HashHeightPair GetIndexedTipHash() + { + var row = this.coinDb.Get(blockTable, blockIndexedHashKey); + if (row != null) + { + var tip = new HashHeightPair(); + tip.FromBytes(row); + return tip; + } + + return null; + } + + private bool TryGetCoins(ReadWriteBatch readWriteBatch, byte[] key, out Coins coins) + { + byte[] row2 = readWriteBatch.Get(coinsTable, key); + if (row2 == null) + { + coins = null; + return false; + } + + coins = this.dBreezeSerializer.Deserialize(row2); + + return true; } private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) { HashHeightPair res = null; - using (var batch = this.coinDb.GetWriteBatch()) + int indexedHeight = this.GetIndexedTipHash()?.Height ?? -1; + + using (var batch = this.coinDb.GetReadWriteBatch()) { + var balanceAdjustments = new Dictionary>(); + for (int height = startHeight; height > (target?.Height ?? (startHeight - 1)) && height > (startHeight - MaxRewindBatchSize); height--) { - byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); + byte[] rowKey = BitConverter.GetBytes(height).Reverse().ToArray(); + byte[] row = this.coinDb.Get(rewindTable, rowKey); if (row == null) throw new InvalidOperationException($"No rewind data found for block at height {height}."); - batch.Delete(rewindTable, BitConverter.GetBytes(height)); + batch.Delete(rewindTable, rowKey); var rewindData = this.dBreezeSerializer.Deserialize(row); foreach (OutPoint outPoint in rewindData.OutputsToRemove) { - this.logger.LogDebug("Outputs of outpoint '{0}' will be removed.", outPoint); - batch.Delete(coinsTable, outPoint.ToBytes()); + byte[] key = outPoint.ToBytes(); + if (this.TryGetCoins(batch, key, out Coins coins)) + { + this.logger.LogDebug("Outputs of outpoint '{0}' will be removed.", outPoint); + + if (height <= indexedHeight) + Update(balanceAdjustments, coins.TxOut.ScriptPubKey, coins.Height, -coins.TxOut.Value); + + batch.Delete(coinsTable, key); + } + else + { + throw new InvalidOperationException(string.Format("Outputs of outpoint '{0}' were not found when attempting removal.", outPoint)); + } } foreach (RewindDataOutput rewindDataOutput in rewindData.OutputsToRestore) { this.logger.LogDebug("Outputs of outpoint '{0}' will be restored.", rewindDataOutput.OutPoint); batch.Put(coinsTable, rewindDataOutput.OutPoint.ToBytes(), this.dBreezeSerializer.Serialize(rewindDataOutput.Coins)); + + if (height <= indexedHeight) + Update(balanceAdjustments, rewindDataOutput.Coins.TxOut.ScriptPubKey, (uint)height, rewindDataOutput.Coins.TxOut.Value); } res = rewindData.PreviousBlockHash; } - this.SetBlockHash(batch, res); + AdjustBalance(batch, balanceAdjustments); + + this.SetBlockHash(batch, res, res.Height < indexedHeight); batch.Write(); } return res; } - public RewindData GetRewindData(int height) + + private void SetBlockHash(IDbBatch batch, HashHeightPair nextBlockHash, bool forceUpdateIndexedHeight = false) { - byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); - return row != null ? this.dBreezeSerializer.Deserialize(row) : null; + this.persistedCoinviewTip = nextBlockHash; + batch.Put(blockTable, blockHashKey, nextBlockHash.ToBytes()); + if (this.BalanceIndexingEnabled || forceUpdateIndexedHeight) + batch.Put(blockTable, blockIndexedHashKey, nextBlockHash.ToBytes()); } - public void PutStake(IEnumerable stakeEntries) + private void EndiannessFix() { - using (var batch = this.coinDb.GetWriteBatch()) + // The fix is still required for fixing up older LevelDb database coming from test zip files. + if (!(this.coinDb is LevelDb levelDb)) + return; + + LevelDB.DB leveldb = (LevelDB.DB)levelDb.GetType().GetField("db", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance).GetValue(levelDb); + + // Check if key bytes are in the wrong endian order. + HashHeightPair current = this.GetTipHash(); + + if (current == null) + return; + + byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(current.Height)); + // Fix the table if required. + if (row != null) { - foreach (StakeItem stakeEntry in stakeEntries) + // To be sure, check the next height too. + byte[] row2 = (current.Height > 1) ? this.coinDb.Get(rewindTable, BitConverter.GetBytes(current.Height - 1)) : new byte[] { }; + if (row2 != null) { - if (!stakeEntry.InStore) + this.logger.LogInformation("Fixing the coin db."); + + var rows = new Dictionary(); + + using (var iterator = leveldb.CreateIterator()) { - batch.Put(stakeTable, stakeEntry.BlockId.ToBytes(false), this.dBreezeSerializer.Serialize(stakeEntry.BlockStake)); - stakeEntry.InStore = true; + iterator.Seek(new byte[] { rewindTable }); + + while (iterator.IsValid()) + { + byte[] key = iterator.Key(); + + if (key.Length != 5 || key[0] != rewindTable) + break; + + int height = BitConverter.ToInt32(key, 1); + + rows[height] = iterator.Value(); + + iterator.Next(); + } } - } - batch.Write(); - } + using (var batch = new LevelDB.WriteBatch()) + { + foreach (int height in rows.Keys.OrderBy(k => k)) + { + batch.Delete(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height)).ToArray()); + } + + foreach (int height in rows.Keys.OrderBy(k => k)) + { + batch.Put(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray(), rows[height]); + } + + leveldb.Write(batch, new LevelDB.WriteOptions() { Sync = true }); + } + } + } } - public void GetStake(IEnumerable blocklist) + private void EnsureCoinDatabaseIntegrity() { - foreach (StakeItem blockStake in blocklist) + this.logger.LogInformation("Checking coin database integrity..."); + + this.EndiannessFix(); + + if (this.GetTipHash() == null) { - this.logger.LogTrace("Loading POS block hash '{0}' from the database.", blockStake.BlockId); - byte[] stakeRow = this.coinDb.Get(stakeTable, blockStake.BlockId.ToBytes(false)); + this.logger.LogInformation($"Rebuilding coin database that has no tip information."); + this.coinDb.Clear(); + return; + } - if (stakeRow != null) + HashHeightPair maxHeight = new HashHeightPair(this.persistedCoinviewTip.Hash, this.persistedCoinviewTip.Height); + + // If the balance table is empty then rebuild the coin db. + if (this.BalanceIndexingEnabled) + { + HashHeightPair indexedTipHash = this.GetIndexedTipHash(); + if (indexedTipHash == null) { - blockStake.BlockStake = this.dBreezeSerializer.Deserialize(stakeRow); - blockStake.InStore = true; + this.logger.LogInformation($"Rebuilding coin database to include balance information."); + this.coinDb.Clear(); + return; } + + if (indexedTipHash.Height < maxHeight.Height) + { + this.logger.LogInformation($"Rewinding the coin database to include missing balance information."); + maxHeight = indexedTipHash; + } + } + + for (int height = this.persistedCoinviewTip.Height; height > maxHeight.Height;) + { + this.logger.LogInformation($"Fixing coin database, deleting rewind data at height {height} above tip '{maxHeight}'."); + + // Do a batch of rewinding. + height = RewindInternal(height, maxHeight).Height; } + + this.logger.LogInformation("Coin database integrity good."); } private void AddBenchStats(StringBuilder log) @@ -395,10 +477,84 @@ private void AddBenchStats(StringBuilder log) this.latestPerformanceSnapShot = snapShot; } + + private void AdjustBalance(ReadWriteBatch batch, Dictionary> balanceUpdates) + { + foreach ((TxDestination txDestination, Dictionary balanceAdjustments) in balanceUpdates) + { + long totalAdjustment = 0; + + foreach (uint height in balanceAdjustments.Keys.OrderBy(k => k)) + { + var key = txDestination.ToBytes().Concat(BitConverter.GetBytes(height).Reverse()).ToArray(); + byte[] row = batch.Get(balanceAdjustmentTable, key); + long adjustment = balanceAdjustments[height]; + long balance = ((row == null) ? 0 : BitConverter.ToInt64(row)) + adjustment; + batch.Put(balanceAdjustmentTable, key, BitConverter.GetBytes(balance)); + + totalAdjustment += adjustment; + } + + { + var key = txDestination.ToBytes(); + byte[] row = batch.Get(balanceTable, key); + long balance = ((row == null) ? 0 : BitConverter.ToInt64(row)) + totalAdjustment; + batch.Put(balanceTable, key, BitConverter.GetBytes(balance)); + } + } + } + + private void Update(Dictionary> balanceAdjustments, Script scriptPubKey, uint height, long change) + { + if (scriptPubKey.Length == 0 || change == 0) + return; + + foreach (TxDestination txDestination in this.scriptAddressReader.GetDestinationFromScriptPubKey(this.network, scriptPubKey)) + { + if (!balanceAdjustments.TryGetValue(txDestination, out Dictionary value)) + { + value = new Dictionary(); + balanceAdjustments[txDestination] = value; + } + + if (!value.TryGetValue(height, out long balance)) + balance = change; + else + balance += change; + + value[height] = balance; + } + } + + /// + public IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination) + { + long balance; + { + byte[] row = this.coinDb.Get(balanceTable, txDestination.ToBytes()); + balance = (row == null) ? 0 : BitConverter.ToInt64(row); + } + + using (var iterator = this.coinDb.GetIterator(balanceAdjustmentTable)) + { + foreach ((uint height, long adjustment) in iterator.GetAll(ascending: false, + lastKey: txDestination.ToBytes().Concat(BitConverter.GetBytes(this.persistedCoinviewTip.Height + 1).Reverse()).ToArray(), + includeLastKey: false, + firstKey: txDestination.ToBytes(), + includeFirstKey: false).Select(x => (height: BitConverter.ToUInt32(x.Item1.Reverse().ToArray()), adjustment: BitConverter.ToInt64(x.Item2)))) + { + yield return (height, balance); + balance -= adjustment; + } + } + + yield return (0, balance); + } + /// public void Dispose() { this.coinDb.Dispose(); } } -} \ No newline at end of file +} diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs index 638444e332..fb5c3aa95d 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs @@ -34,6 +34,8 @@ public class DBreezeCoindb : ICoindb, IStakedb, IDisposable private BackendPerformanceSnapshot latestPerformanceSnapShot; + public bool BalanceIndexingEnabled => false; + /// Access to dBreeze database. private readonly DBreezeEngine dBreeze; @@ -65,7 +67,7 @@ public DBreezeCoindb(Network network, string folder, IDateTimeProvider dateTimeP nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 300); } - public void Initialize(ChainedHeader chainTip) + public void Initialize(bool balanceIndexingEnabled) { Block genesis = this.network.GetGenesis(); @@ -97,6 +99,11 @@ public HashHeightPair GetTipHash() return tipHash; } + public IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination) + { + throw new NotImplementedException(); + } + public FetchCoinsResponse FetchCoins(OutPoint[] utxos) { FetchCoinsResponse res = new FetchCoinsResponse(); @@ -150,7 +157,7 @@ private void SetBlockHash(DBreeze.Transactions.Transaction transaction, HashHeig transaction.Insert("BlockHash", blockHashKey, nextBlockHash.ToBytes()); } - public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) + public void SaveChanges(IList unspentOutputs, Dictionary> balanceUpdates, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) { int insertedEntities = 0; @@ -302,10 +309,7 @@ public HashHeightPair Rewind(HashHeightPair target) return res; } - /// - /// Persists unsaved POS blocks information to the database. - /// - /// List of POS block information to be examined and persists if unsaved. + /// public void PutStake(IEnumerable stakeEntries) { using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction()) diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs index f8c664d6b6..705d4679bc 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs @@ -10,8 +10,10 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews public interface ICoindb { /// Initialize the coin database. - /// The current chain's tip. - void Initialize(ChainedHeader chainTip); + /// Indicates whether to enable balance indexing. + void Initialize(bool balanceIndexingEnabled); + + bool BalanceIndexingEnabled { get; } /// /// Retrieves the block hash of the current tip of the coinview. @@ -31,10 +33,11 @@ public interface ICoindb /// /// Information about the changes between the old block and the new block. An item in this list represents a list of all outputs /// for a specific transaction. If a specific output was spent, the output is null. + /// Non-cumulative balance updates at each height. /// Block hash of the current tip of the coinview. /// Block hash of the tip of the coinview after the change is applied. /// List of rewind data items to be persisted. - void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null); + void SaveChanges(IList unspentOutputs, Dictionary> balanceUpdates, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null); /// /// Obtains information about unspent outputs. @@ -69,13 +72,15 @@ public interface ICoindb /// See . RewindData GetRewindData(int height); - /// Gets the minimum rewind height. - /// - /// - /// The minimum rewind height or -1 if rewind is not possible. - /// - /// - int GetMinRewindHeight(); + /// + /// Returns a combination of (height, satoshis) values with the cumulative balance up to the corresponding height. + /// + /// The destination value derived from the address being queried. + /// A combination of (height, satoshis) values with the cumulative balance up to the corresponding height. + /// Balance updates (even when nett 0) are delivered for every height at which transactions for the address + /// had been recorded and as such the returned heights can be used in conjunction with the block store to discover + /// all related transactions. + IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination); } public interface IStakedb : ICoindb diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs index 9cb80e7129..4332bf0800 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs @@ -24,6 +24,8 @@ public class InMemoryCoinView : ICoinView, ICoindb /// All access to this object has to be protected by . private HashHeightPair tipHash; + public bool BalanceIndexingEnabled => false; + /// /// Initializes an instance of the object. /// @@ -39,6 +41,11 @@ public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConse throw new NotImplementedException(); } + /// + public void Initialize(bool balanceIndexingEnabled) + { + } + /// public void Sync(ChainIndexer chainIndexer) { @@ -55,6 +62,11 @@ public void CacheCoins(OutPoint[] utxos) throw new NotImplementedException(); } + public IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination) + { + throw new NotImplementedException(); + } + /// public FetchCoinsResponse FetchCoins(OutPoint[] txIds) { @@ -107,6 +119,11 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB } } + public void SaveChanges(IList unspentOutputs, Dictionary> balanceUpdates, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) + { + this.SaveChanges(unspentOutputs, oldBlockHash, nextBlockHash, rewindDataList); + } + public int GetMinRewindHeight() { throw new NotImplementedException(); diff --git a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs index 06a751fb35..2845734072 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs @@ -81,6 +81,8 @@ public static IFullNodeBuilder UsePosConsensus(this IFullNodeBuilder fullNodeBui public static void ConfigureCoinDatabaseImplementation(this IServiceCollection services, DbType coindbType) { + services.Replace((p, old) => old ?? new ScriptAddressReader(), ServiceLifetime.Singleton); + switch (coindbType) { case DbType.Dbreeze: diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs index 79e782b89e..68aaed1d69 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs @@ -85,6 +85,11 @@ public FetchCoinsResponse FetchCoins(OutPoint[] txIds) { throw new NotImplementedException(); } + + public IEnumerable<(uint, long)> GetBalance(TxDestination txDestination) + { + throw new NotImplementedException(); + } public HashHeightPair Rewind(HashHeightPair target) { diff --git a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs index dc3ae155a1..5a6d28e632 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs @@ -46,14 +46,18 @@ public void TestDBreezeSerialization() Block genesis = ctx.Network.GetGenesis(); var genesisChainedHeader = new ChainedHeader(genesis.Header, ctx.Network.GenesisHash, 0); ChainedHeader chained = this.MakeNext(genesisChainedHeader, ctx.Network); - ctx.Coindb.SaveChanges(new UnspentOutput[] { new UnspentOutput(new OutPoint(genesis.Transactions[0], 0), new Coins(0, genesis.Transactions[0].Outputs.First(), true)) }, new HashHeightPair(genesisChainedHeader), new HashHeightPair(chained)); + var coins = new Coins(0, genesis.Transactions[0].Outputs.First(), true); + TxDestination txDestination = PayToPubkeyTemplate.Instance.ExtractScriptPubKeyParameters(coins.TxOut.ScriptPubKey).Hash; + ctx.Coindb.SaveChanges(new UnspentOutput[] { new UnspentOutput(new OutPoint(genesis.Transactions[0], 0), coins) }, + new Dictionary>() { { txDestination, new Dictionary { { 0, coins.TxOut.Value } } } }, + new HashHeightPair(genesisChainedHeader), new HashHeightPair(chained)); Assert.NotNull(ctx.Coindb.FetchCoins(new[] { new OutPoint(genesis.Transactions[0], 0) }).UnspentOutputs.Values.FirstOrDefault().Coins); Assert.Null(ctx.Coindb.FetchCoins(new[] { new OutPoint() }).UnspentOutputs.Values.FirstOrDefault().Coins); ChainedHeader previous = chained; chained = this.MakeNext(this.MakeNext(genesisChainedHeader, ctx.Network), ctx.Network); chained = this.MakeNext(this.MakeNext(genesisChainedHeader, ctx.Network), ctx.Network); - ctx.Coindb.SaveChanges(new List(), new HashHeightPair(previous), new HashHeightPair(chained)); + ctx.Coindb.SaveChanges(new List(), new Dictionary>(), new HashHeightPair(previous), new HashHeightPair(chained)); Assert.Equal(chained.HashBlock, ctx.Coindb.GetTipHash().Hash); ctx.ReloadPersistentCoinView(chained); Assert.Equal(chained.HashBlock, ctx.Coindb.GetTipHash().Hash); diff --git a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs index fb005f0b45..7cbea9d885 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs @@ -5,6 +5,7 @@ using Moq; using NBitcoin; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.Consensus.CoinViews; using Stratis.Bitcoin.Interfaces; @@ -28,8 +29,8 @@ public NodeContext(object caller, string name, Network network) this.FolderName = TestBase.CreateTestDir(caller, name); var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - this.Coindb = new Coindb(network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer); - this.Coindb.Initialize(new ChainedHeader(network.GetGenesis().Header, network.GenesisHash, 0)); + this.Coindb = new Coindb(network, new DataFolder(this.FolderName), dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer, new ScriptAddressReader()); + this.Coindb.Initialize(false); this.cleanList = new List { (IDisposable)this.Coindb }; } @@ -66,9 +67,9 @@ public void ReloadPersistentCoinView(ChainedHeader chainTip) this.cleanList.Remove((IDisposable)this.Coindb); var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - this.Coindb = new Coindb(this.Network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer); + this.Coindb = new Coindb(this.Network, new DataFolder(this.FolderName), dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer, new ScriptAddressReader()); - this.Coindb.Initialize(chainTip); + this.Coindb.Initialize(false); this.cleanList.Add((IDisposable)this.Coindb); } } diff --git a/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs b/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs index 4c8f8bcfa6..0d0136fb2f 100644 --- a/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs @@ -75,6 +75,11 @@ public FetchCoinsResponse FetchCoins(OutPoint[] txIds) } } + public IEnumerable<(uint, long)> GetBalance(TxDestination txDestination) + { + throw new NotImplementedException(); + } + /// public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) { diff --git a/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs b/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs new file mode 100644 index 0000000000..b84a330aa8 --- /dev/null +++ b/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using NBitcoin; + +namespace Stratis.Bitcoin.Database +{ + /// + /// Extension methods that build on the interface. + /// + public static class IDbIteratorExt + { + private static ByteArrayComparer byteArrayComparer = new ByteArrayComparer(); + + /// + /// Gets all the keys in the relevant table subject to any supplied constraints. + /// + /// The iterator that also identifies the table being iterated. + /// Defaults to false. Set to true if values should be ommitted - i.e. set to null. + /// Defaults to true. Set to false to return keys in ascending order. + /// Can be set optionally to specify the lower bound of keys to return. + /// Can be set optionally to specify the upper bound of keys to return. + /// Defaults to true. Set to false to omit the key specified in . + /// Defaults to true. Set to false to omit the key specified in . + /// An enumeration containing all the keys and values according to the specified constraints. + public static IEnumerable<(byte[], byte[])> GetAll(this IDbIterator iterator, bool keysOnly = false, bool ascending = true, + byte[] firstKey = null, byte[] lastKey = null, bool includeFirstKey = true, bool includeLastKey = true) + { + bool done = false; + Func breakLoop; + Action next; + + if (!ascending) + { + // Seek to the last key if it was provided. + if (lastKey == null) + iterator.SeekToLast(); + else + { + iterator.Seek(lastKey); + if (!(includeLastKey && iterator.IsValid() && byteArrayComparer.Equals(iterator.Key(), lastKey))) + iterator.Prev(); + } + + breakLoop = (firstKey == null) ? (Func)null : (keyBytes) => + { + int compareResult = byteArrayComparer.Compare(keyBytes, firstKey); + if (compareResult <= 0) + { + // If this is the first key and its not included or we've overshot the range then stop without yielding a value. + if (!includeFirstKey || compareResult < 0) + return true; + + // Stop after yielding the value. + done = true; + } + + // Keep going. + return false; + }; + + next = () => iterator.Prev(); + } + else /* Ascending */ + { + // Seek to the first key if it was provided. + if (firstKey == null) + iterator.Seek(new byte[0]); + else + { + iterator.Seek(firstKey); + if (!(includeFirstKey && iterator.IsValid() && byteArrayComparer.Equals(iterator.Key(), firstKey))) + iterator.Next(); + } + + breakLoop = (lastKey == null) ? (Func)null : (keyBytes) => + { + int compareResult = byteArrayComparer.Compare(keyBytes, lastKey); + if (compareResult >= 0) + { + // If this is the last key and its not included or we've overshot the range then stop without yielding a value. + if (!includeLastKey || compareResult > 0) + return true; + + // Stop after yielding the value. + done = true; + } + + // Keep going. + return false; + }; + + next = () => iterator.Next(); + } + + while (iterator.IsValid()) + { + byte[] keyBytes = iterator.Key(); + + if (breakLoop != null && breakLoop(keyBytes)) + break; + + yield return (keyBytes, keysOnly ? null : iterator.Value()); + + if (done) + break; + + next(); + } + } + } +} diff --git a/src/Stratis.Bitcoin/Database/ReadWriteBatch.cs b/src/Stratis.Bitcoin/Database/ReadWriteBatch.cs new file mode 100644 index 0000000000..4843eb969e --- /dev/null +++ b/src/Stratis.Bitcoin/Database/ReadWriteBatch.cs @@ -0,0 +1,94 @@ +using System.Collections.Generic; +using System.Linq; +using NBitcoin; + +namespace Stratis.Bitcoin.Database +{ + /// + /// A batch that can be used to record changes that can be applied atomically. + /// + /// The supplied method will immediately reflect any changes that have + /// been made or retrieve the value from the underlying database. In contrast the database method + /// will only show the changes after the method is called. + public class ReadWriteBatch : IDbBatch + { + private readonly IDb db; + private readonly IDbBatch batch; + private Dictionary cache; + + public ReadWriteBatch(IDb db) + { + this.db = db; + this.batch = db.GetWriteBatch(); + this.cache = new Dictionary(new ByteArrayComparer()); + } + + /// + /// Records a value that will be written to the database when the method is invoked. + /// + /// The table that will be updated. + /// The table key that identifies the value to be updated. + /// The value to be written to the table. + /// This class for fluent operations. + public IDbBatch Put(byte table, byte[] key, byte[] value) + { + this.cache[new byte[] { table }.Concat(key).ToArray()] = value; + return this.batch.Put(table, key, value); + } + + /// + /// Records a key that will be deleted from the database when the method is invoked. + /// + /// The table that will be updated. + /// The table key that will be removed. + /// This interface for fluent operations. + public IDbBatch Delete(byte table, byte[] key) + { + this.cache[new byte[] { table }.Concat(key).ToArray()] = null; + return this.batch.Delete(table, key); + } + + /// + /// Returns any changes that have been made to the batch or retrieves the value from the underlying database.. + /// + /// The table of the value to be retrieved. + /// The table key of the value to retrieve. + /// This interface for fluent operations. + public byte[] Get(byte table, byte[] key) + { + if (this.cache.TryGetValue(new byte[] { table }.Concat(key).ToArray(), out byte[] value)) + return value; + + return this.db.Get(table, key); + } + + /// + /// Writes the recorded changes to the database. + /// + public void Write() + { + this.batch.Write(); + } + + public void Dispose() + { + this.batch.Dispose(); + } + } + + /// + /// Extension methods that build on the interface. + /// + public static class IDbExt + { + /// + /// Gets a . + /// + /// The database to get the batch for. + /// The . + public static ReadWriteBatch GetReadWriteBatch(this IDb db) + { + return new ReadWriteBatch(db); + } + } +} diff --git a/src/Stratis.Bitcoin/Database/RocksDb.cs b/src/Stratis.Bitcoin/Database/RocksDb.cs index f392c7e3b4..e2adcfd6eb 100644 --- a/src/Stratis.Bitcoin/Database/RocksDb.cs +++ b/src/Stratis.Bitcoin/Database/RocksDb.cs @@ -106,7 +106,7 @@ public void Prev() public bool IsValid() { - return this.iterator.Valid() && this.iterator.Value()[0] == this.table; + return this.iterator.Valid() && this.iterator.Key()[0] == this.table; } public byte[] Key()