[DO-NOT-MERGE] Leios prototype

CODEOWNERS

@@ -1,6 +1,7 @@
 # catch-all
 
-* @intersectmbo/cardano-node-maintainers
+# FIXME: codeowners disabled for leios-prototype
+# * @intersectmbo/cardano-node-maintainers
 
 # Specific reviewers for code pieces
 # NB - The last matching pattern takes precedence

cabal.project

@@ -96,3 +96,38 @@ if impl (ghc >= 9.12)
 
 constraints:
     hedgehog-extras == 0.7.0.0
+
+-- Points to ouroboros-consensus/leios-prototype
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-consensus
+  tag: 8f0e1423c99bcfa0025ae5476a5467a4af0d2828
+  --sha256: sha256-/1hOntoPR10b7e2NogYzlN0qaBeuXk3O1UbGi9MwkBI=
+  subdir:
+    ouroboros-consensus
+    ouroboros-consensus-cardano
+    ouroboros-consensus-protocol
+    ouroboros-consensus-diffusion
+    ouroboros-consensus-observe
+    sop-extras
+    strict-sop-core
+
+-- Points to ouroboros-network/leios-prototype
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-network.git
+  tag: fb5f16f48af84950db0c2ecefd8cbda532c77d1a
+  --sha256: sha256-0LWD1zwOzhdcPJtBuw7pRDVcBd9CKJOFjm9o4BvsIMM=
+  subdir:
+    cardano-ping
+    monoidal-synchronisation
+    quickcheck-monoids
+    network-mux
+    ouroboros-network
+    ouroboros-network-api
+    ouroboros-network-framework
+    ouroboros-network-mock
+    ouroboros-network-protocols
+    ouroboros-network-testing
+    ntp-client
+    cardano-client

cardano-node/cardano-node.cabal

@@ -197,6 +197,7 @@ library
                       , ouroboros-consensus-cardano ^>= 0.25
                       , ouroboros-consensus-diffusion ^>= 0.23
                       , ouroboros-consensus-protocol
+                      , ouroboros-consensus-observe
                       , ouroboros-network-api ^>= 0.14.1
                       , ouroboros-network ^>= 0.21.3
                       , ouroboros-network-framework ^>= 0.18

cardano-node/src/Cardano/Node/Run.hs

@@ -169,6 +169,8 @@ import           Paths_cardano_node (version)
 
 import           Paths_cardano_node (version)
 
+import           LeiosDemoTypes (demoNewLeiosDbConnectionIO)
+
 {- HLINT ignore "Fuse concatMap/map" -}
 {- HLINT ignore "Redundant <$>" -}
 {- HLINT ignore "Use fewer imports" -}
@@ -520,6 +522,7 @@ handleSimpleNode blockType runP p2pMode tracers nc onKernel = do
               , rnEnableP2P      = p2pMode
               , rnPeerSharing    = ncPeerSharing nc
               , rnGetUseBootstrapPeers = readTVar useBootstrapVar
+              , rnNewLeiosDbConnection = demoNewLeiosDbConnectionIO
               }
 #ifdef UNIX
         -- initial `SIGHUP` handler, which only rereads the topology file but
@@ -618,6 +621,7 @@ handleSimpleNode blockType runP p2pMode tracers nc onKernel = do
                 , rnEnableP2P      = p2pMode
                 , rnPeerSharing    = ncPeerSharing nc
                 , rnGetUseBootstrapPeers = pure DontUseBootstrapPeers
+                , rnNewLeiosDbConnection = demoNewLeiosDbConnectionIO
                 }
 #ifdef UNIX
         -- initial `SIGHUP` handler; it only warns that neither updating of

cardano-node/src/Cardano/Node/TraceConstraints.hs

@@ -15,14 +15,17 @@ import           Cardano.Node.Queries (ConvertTxId, GetKESInfo (..), HasKESInfo
 import           Cardano.Protocol.Crypto (StandardCrypto)
 import           Cardano.Tracing.HasIssuer (HasIssuer)
 import           Ouroboros.Consensus.Block (BlockProtocol, CannotForge, ForgeStateUpdateError,
-                   GetHeader, HasHeader, Header)
+                   GetHeader, HasHeader, Header, HeaderHash)
 import           Ouroboros.Consensus.HeaderValidation (OtherHeaderEnvelopeError)
 import           Ouroboros.Consensus.Ledger.Abstract (LedgerError)
 import           Ouroboros.Consensus.Ledger.Inspect (LedgerEvent, LedgerUpdate, LedgerWarning)
-import           Ouroboros.Consensus.Ledger.SupportsMempool (ApplyTxErr, HasTxId, HasTxs (..))
+import           Ouroboros.Consensus.Ledger.SupportsMempool (ApplyTxErr, HasTxId, HasTxs (..),
+                   TxMeasure)
 import           Ouroboros.Consensus.Node.NetworkProtocolVersion
                    (HasNetworkProtocolVersion (BlockNodeToClientVersion, BlockNodeToNodeVersion))
 import           Ouroboros.Consensus.Node.Run (RunNode, SerialiseNodeToNodeConstraints)
+import           Ouroboros.Consensus.Node.Tracers (ForgedBlock)
+import           Ouroboros.Consensus.Observe.ConsensusJson (ConsensusJson)
 import           Ouroboros.Consensus.Protocol.Abstract (SelectView, ValidationErr)
 import           Ouroboros.Consensus.Shelley.Ledger.Mempool (GenTx, TxId)
 import           Ouroboros.Network.Block (Serialised)
@@ -58,7 +61,6 @@ type TraceConstraints blk =
 
     , ToJSON (BlockNodeToClientVersion blk)
     , ToJSON (BlockNodeToNodeVersion blk)
-
     , LogFormatting (ApplyTxErr blk)
     , LogFormatting (GenTx blk)
     , LogFormatting (Header blk)
@@ -72,4 +74,7 @@ type TraceConstraints blk =
     , LogFormatting (ForgeStateUpdateError blk)
     , LogFormatting (Set (Credential 'Staking))
     , LogFormatting (NonEmpty.NonEmpty (KeyHash 'Staking))
+
+    , ConsensusJson (HeaderHash blk)
+    , ConsensusJson (TxMeasure blk)
     )

cardano-node/src/Cardano/Node/Tracing/Consistency.hs

@@ -101,6 +101,9 @@ import qualified Data.Text as T
 import qualified Network.Mux as Mux
 import qualified Network.Socket as Socket
 
+import           LeiosDemoTypes (LeiosPoint, LeiosEb, LeiosTx, TraceLeiosKernel, TraceLeiosPeer)
+import           LeiosDemoOnlyTestFetch (LeiosFetch)
+import           LeiosDemoOnlyTestNotify (LeiosNotify)
 
 -- | Check the configuration in the given file.
 -- If there is no configuration in the file check the standard configuration
@@ -207,7 +210,10 @@ getAllNamespaces =
                         (allNamespaces :: [Namespace (Jumping.TraceEventCsj peer blk)])
         dbfNS = map (nsGetTuple . nsReplacePrefix  ["Consensus", "DevotedBlockFetch"])
                         (allNamespaces :: [Namespace (Jumping.TraceEventDbf peer)])
-
+        leiosKernelNS = map (nsGetTuple . nsReplacePrefix  ["Consensus", "LeiosKernel"])
+                        (allNamespaces :: [Namespace TraceLeiosKernel])
+        leiosPeerNS = map (nsGetTuple . nsReplacePrefix  ["Consensus", "LeiosPeer"])
+                        (allNamespaces :: [Namespace (BlockFetch.TraceLabelPeer remotePeer TraceLeiosPeer)])
 -- Node to client
         keepAliveClientNS = map (nsGetTuple . nsReplacePrefix ["Net"])
                                 (allNamespaces :: [Namespace  (TraceKeepAliveClient peer)])
@@ -263,6 +269,18 @@ getAllNamespaces =
                                     (TraceSendRecv
                                       (TxSubmission2 (GenTxId blk) (GenTx blk))))])
 
+        leiosNotifyNS = map (nsGetTuple . nsReplacePrefix ["LeiosNotify", "Remote"])
+                             (allNamespaces :: [Namespace
+                                 (BlockFetch.TraceLabelPeer peer
+                                    (TraceSendRecv
+                                      (LeiosNotify LeiosPoint ())))])
+
+        leiosFetchNS = map (nsGetTuple . nsReplacePrefix ["LeiosFetch", "Remote"])
+                             (allNamespaces :: [Namespace
+                                 (BlockFetch.TraceLabelPeer peer
+                                    (TraceSendRecv
+                                      (LeiosFetch LeiosPoint LeiosEb LeiosTx)))])
+
 -- Diffusion
 
         dtMuxNS = map (nsGetTuple . nsReplacePrefix ["Net", "Mux", "Remote"])
@@ -420,6 +438,8 @@ getAllNamespaces =
             <> gsmNS
             <> csjNS
             <> dbfNS
+            <> leiosKernelNS
+            <> leiosPeerNS
 -- NodeToClient
             <> keepAliveClientNS
             <> chainSyncNS
@@ -432,6 +452,8 @@ getAllNamespaces =
             <> blockFetchNS
             <> blockFetchSerialisedNS
             <> txSubmission2NS
+            <> leiosNotifyNS
+            <> leiosFetchNS
 -- Diffusion
             <> dtMuxNS
             <> dtLocalMuxNS

cardano-node/src/Cardano/Node/Tracing/Tracers.hs

@@ -362,6 +362,16 @@ mkConsensusTracers configReflection trBase trForward mbTrEKG _trDataPoint trConf
                 ["Consensus", "DevotedBlockFetch"]
     configureTracers configReflection trConfig [consensusDbfTr]
 
+    !consensusLeiosKernelTr <- mkCardanoTracer
+                trBase trForward mbTrEKG
+                ["Consensus", "LeiosKernel"]
+    configureTracers configReflection trConfig [consensusLeiosKernelTr]
+
+    !consensusLeiosPeerTr <- mkCardanoTracer
+                trBase trForward mbTrEKG
+                ["Consensus", "LeiosPeer"]
+    configureTracers configReflection trConfig [consensusLeiosPeerTr]
+
     pure $ Consensus.Tracers
       { Consensus.chainSyncClientTracer = Tracer $
           traceWith chainSyncClientTr
@@ -408,6 +418,10 @@ mkConsensusTracers configReflection trBase trForward mbTrEKG _trDataPoint trConf
           traceWith consensusCsjTr
       , Consensus.dbfTracer = Tracer $
           traceWith consensusDbfTr
+      , Consensus.leiosKernelTracer = Tracer $
+          traceWith consensusLeiosKernelTr
+      , Consensus.leiosPeerTracer = Tracer $
+          traceWith consensusLeiosPeerTr
       }
 
 mkNodeToClientTracers :: forall blk.
@@ -502,6 +516,16 @@ mkNodeToNodeTracers configReflection trBase trForward mbTrEKG _trDataPoint trCon
                 ["PeerSharing", "Remote"]
     configureTracers configReflection trConfig [peerSharingTracer]
 
+    !leiosNotifyTracer  <-  mkCardanoTracer
+                trBase trForward mbTrEKG
+                ["LeiosNotify", "Remote"]
+    configureTracers configReflection trConfig [leiosNotifyTracer]
+
+    !leiosFetchTracer  <-  mkCardanoTracer
+                trBase trForward mbTrEKG
+                ["LeiosFetch", "Remote"]
+    configureTracers configReflection trConfig [leiosFetchTracer]
+
     pure $ NtN.Tracers
       { NtN.tChainSyncTracer = Tracer $
           traceWith chainSyncTracer
@@ -517,6 +541,10 @@ mkNodeToNodeTracers configReflection trBase trForward mbTrEKG _trDataPoint trCon
           traceWith keepAliveTracer
       , NtN.tPeerSharingTracer = Tracer $
           traceWith peerSharingTracer
+      , NtN.tLeiosNotifyTracer = Tracer $
+          traceWith leiosNotifyTracer
+      , NtN.tLeiosFetchTracer = Tracer $
+          traceWith leiosFetchTracer
       }
 
 mkDiffusionTracers

cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs

@@ -40,9 +40,12 @@ import           Ouroboros.Consensus.Genesis.Governor (DensityBounds (..), GDDDe
 import           Ouroboros.Consensus.Ledger.Extended (ExtValidationError)
 import           Ouroboros.Consensus.Ledger.Inspect (LedgerEvent (..), LedgerUpdate, LedgerWarning)
 import           Ouroboros.Consensus.Ledger.SupportsMempool (ApplyTxErr, ByteSize32 (..), GenTxId,
-                   HasTxId, LedgerSupportsMempool, txForgetValidated, txId)
+                   HasTxId, LedgerSupportsMempool, TxLimits (TxMeasure),
+                   TxMeasureMetrics (txMeasureMetricExUnitsMemory, txMeasureMetricExUnitsSteps, txMeasureMetricRefScriptsSizeBytes, txMeasureMetricTxSizeBytes),
+                   txForgetValidated, txId)
 import           Ouroboros.Consensus.Ledger.SupportsProtocol
 import           Ouroboros.Consensus.Mempool (MempoolSize (..), TraceEventMempool (..))
+import           Ouroboros.Consensus.Mempool.TxSeq (TxSeqMeasure (mCount, mSize))
 import           Ouroboros.Consensus.MiniProtocol.BlockFetch.Server
                    (TraceBlockFetchServerEvent (..))
 import           Ouroboros.Consensus.MiniProtocol.ChainSync.Client
@@ -54,6 +57,7 @@ import           Ouroboros.Consensus.MiniProtocol.LocalTxSubmission.Server
 import           Ouroboros.Consensus.Node.GSM
 import           Ouroboros.Consensus.Node.Run (SerialiseNodeToNodeConstraints, estimateBlockSize)
 import           Ouroboros.Consensus.Node.Tracers
+import           Ouroboros.Consensus.Observe.ConsensusJson (ConsensusJson (toConsensusJson))
 import qualified Ouroboros.Consensus.Protocol.Ledger.HotKey as HotKey
 import           Ouroboros.Consensus.Util.Enclose
 import qualified Ouroboros.Network.AnchoredFragment as AF
@@ -81,6 +85,8 @@ import           Data.Time (NominalDiffTime)
 import           Data.Word (Word32, Word64)
 import           Network.TypedProtocol.Core
 
+import           LeiosDemoTypes (TraceLeiosKernel (..), TraceLeiosPeer, leiosEbTxs,
+                   traceLeiosKernelToObject, traceLeiosPeerToObject)
 
 instance (LogFormatting adr, Show adr) => LogFormatting (ConnectionId adr) where
   forMachine _dtal (ConnectionId local' remote) =
@@ -1446,7 +1452,10 @@ instance ( tx ~ GenTx blk
          , Show (TxId (GenTx blk))
          , LogFormatting (CannotForge blk)
          , LogFormatting (ExtValidationError blk)
-         , LogFormatting (ForgeStateUpdateError blk))
+         , LogFormatting (ForgeStateUpdateError blk)
+         , ConsensusJson (HeaderHash blk)
+         , ConsensusJson (TxMeasure blk)
+         )
       => LogFormatting (TraceForgeEvent blk) where
   forMachine _dtal (TraceStartLeadershipCheck slotNo) =
     mconcat
@@ -1529,16 +1538,11 @@ instance ( tx ~ GenTx blk
       , "mempoolHash" .= String (renderChainHash @blk (renderHeaderHash (Proxy @blk)) mpHash)
       , "mempoolSlot" .= toJSON (unSlotNo mpSlot)
       ]
-  forMachine _dtal (TraceForgedBlock slotNo _ blk _) =
+  forMachine _dtal (TraceForgedBlock slotNo fb) =
     mconcat
       [ "kind" .= String "TraceForgedBlock"
       , "slot" .= toJSON (unSlotNo slotNo)
-      , "block"     .= String (renderHeaderHash (Proxy @blk) $ blockHash blk)
-      , "blockNo"   .= toJSON (unBlockNo $ blockNo blk)
-      , "blockPrev" .= String (renderChainHash
-                                @blk
-                                (renderHeaderHash (Proxy @blk))
-                                $ blockPrevHash blk)
+      , "forgedBlock"     .= toConsensusJson fb
       ]
   forMachine _dtal (TraceDidntAdoptBlock slotNo _) =
     mconcat
@@ -1641,7 +1645,7 @@ instance ( tx ~ GenTx blk
         <> renderChainHash @blk (renderHeaderHash (Proxy @blk)) mpHash
         <> " ticked to slot "
         <> showT (unSlotNo mpSlot)
-  forHuman (TraceForgedBlock slotNo _ _ _) =
+  forHuman (TraceForgedBlock slotNo _) =
       "Forged block in slot " <> showT (unSlotNo slotNo)
   forHuman (TraceDidntAdoptBlock slotNo _) =
       "Didn't adopt forged block in slot " <> showT (unSlotNo slotNo)
@@ -1699,9 +1703,22 @@ instance ( tx ~ GenTx blk
     [CounterM "Forge.node-is-leader" Nothing]
   asMetrics TraceForgeTickedLedgerState {} = []
   asMetrics TraceForgingMempoolSnapshot {} = []
-  asMetrics (TraceForgedBlock slot _ _ _) =
-    [IntM "forgedSlotLast" (fromIntegral $ unSlotNo slot),
-     CounterM "Forge.forged" Nothing]
+  asMetrics (TraceForgedBlock slot fb) =
+      [ IntM "forgedSlotLast" (fromIntegral $ unSlotNo slot)
+      , CounterM "Forge.forged" Nothing
+      -- NOTE(bladyjoker): New!!!
+      , CounterM "Forge.ranking-block.total-count" Nothing
+      , CounterM "Forge.ranking-block.total-tx-count" (Just . fromInteger . toInteger . mCount . fbNewBlockSize $ fb)
+      , CounterM "Forge.ranking-block.total-tx-bytes" (Just . fromInteger . toInteger . unByteSize32 . txMeasureMetricTxSizeBytes . mSize . fbNewBlockSize $ fb)
+      , CounterM "Forge.ranking-block.total-tx-xu-memory" (Just . fromInteger . toInteger . txMeasureMetricExUnitsMemory . mSize . fbNewBlockSize $ fb)
+      , CounterM "Forge.ranking-block.total-tx-xu-time" (Just . fromInteger . toInteger . txMeasureMetricExUnitsSteps . mSize . fbNewBlockSize $ fb)
+      , CounterM "Forge.ranking-block.total-tx-ref-script-size-bytes" (Just . fromInteger . toInteger . unByteSize32 . txMeasureMetricRefScriptsSizeBytes . mSize . fbNewBlockSize $ fb)
+      , CounterM "Forge.rest-in-mempool.total-tx-count" (Just . fromInteger . toInteger . mCount . fbMempoolRestSize $ fb)
+      , CounterM "Forge.rest-in-mempool.total-tx-bytes" (Just . fromInteger . toInteger . unByteSize32 . txMeasureMetricTxSizeBytes . mSize . fbMempoolRestSize $ fb)
+      , CounterM "Forge.rest-in-mempool.total-tx-xu-memory" (Just . fromInteger . toInteger . txMeasureMetricExUnitsMemory . mSize . fbMempoolRestSize $ fb)
+      , CounterM "Forge.rest-in-mempool.total-tx-xu-time" (Just . fromInteger . toInteger . txMeasureMetricExUnitsSteps . mSize . fbMempoolRestSize $ fb)
+      , CounterM "Forge.rest-in-mempool.total-tx-ref-script-size-bytes" (Just . fromInteger . toInteger . unByteSize32 . txMeasureMetricRefScriptsSizeBytes . mSize . fbMempoolRestSize $ fb)
+      ]
   asMetrics (TraceDidntAdoptBlock _slot _) =
     [CounterM "Forge.didnt-adopt" Nothing]
   asMetrics (TraceForgedInvalidBlock _slot _ _) =
@@ -2267,3 +2284,36 @@ instance ( StandardHash blk
             ]
 
   forHuman = showT
+
+-----
+
+instance LogFormatting TraceLeiosKernel where
+  forHuman = showT
+  forMachine _dtal = traceLeiosKernelToObject
+
+  asMetrics (TraceLeiosBlockForged{eb, ebMeasure}) =
+      [ CounterM "Forge.endorser-block.total-count" Nothing
+      ,
+        CounterM "Forge.endorser-block.total-tx-count" (Just . fromIntegral . length $ leiosEbTxs eb)
+        , CounterM "Forge.endorser-block.total-tx-bytes" (Just . fromInteger . toInteger . unByteSize32 . txMeasureMetricTxSizeBytes $ ebMeasure)
+        , CounterM "Forge.endorser-block.total-tx-xu-memory" (Just . fromInteger . toInteger . txMeasureMetricExUnitsMemory $ ebMeasure)
+        , CounterM "Forge.endorser-block.total-tx-xu-time" (Just . fromInteger . toInteger . txMeasureMetricExUnitsSteps $ ebMeasure)
+        , CounterM "Forge.endorser-block.total-tx-ref-script-size-bytes" (Just . fromInteger . toInteger . unByteSize32 . txMeasureMetricRefScriptsSizeBytes $ ebMeasure)
+      ]
+  asMetrics _ = []
+
+instance MetaTrace TraceLeiosKernel where
+  namespaceFor _ = Namespace [] []
+  severityFor _ _ = Just Debug
+  documentFor _ = Nothing
+  allNamespaces = [ Namespace [] [] ]
+
+instance LogFormatting TraceLeiosPeer where
+  forHuman = showT
+  forMachine _dtal = traceLeiosPeerToObject
+
+instance MetaTrace TraceLeiosPeer where
+  namespaceFor _ = Namespace [] []
+  severityFor _ _ = Just Debug
+  documentFor _ = Nothing
+  allNamespaces = [ Namespace [] [] ]