feat: agent

.gitignore

@@ -6,6 +6,7 @@ obj/
 #*.idea
 .idea/.idea.Drift/.idea/watcherTasks.xml
 .idea/.idea.Drift/.idea/encodings.xml
+.idea/.idea.Drift.Build/.idea/encodings.xml
 *.DotSettings
 *.received.*
 artifacts/
@@ -14,4 +15,5 @@ TestResults/
 build.binlog
 build.binlog-warnings-only.log
 publish.binlog
-publish.binlog-warnings-only.log
+publish.binlog-warnings-only.log
+containerlab/*/

Directory.Packages.props

@@ -5,6 +5,12 @@
     <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
   </PropertyGroup>
   <ItemGroup>
+    <PackageVersion Include="Google.Protobuf" Version="3.33.5" />
+    <PackageVersion Include="Grpc.AspNetCore" Version="2.76.0" />
+    <PackageVersion Include="Grpc.Core.Api" Version="2.76.0" />
+    <PackageVersion Include="Grpc.Net.Client" Version="2.76.0" />
+    <PackageVersion Include="Grpc.Tools" Version="2.78.0" />
+    <PackageVersion Include="HLabs.Containers" Version="1.0.0-preview.1" />
     <PackageVersion Include="HLabs.ImageReferences" Version="1.0.0-preview.3" />
     <PackageVersion Include="HLabs.ImageReferences.Extensions.Nuke" Version="1.0.0-preview.3" />
     <PackageVersion Include="Humanizer" Version="3.0.1" />
@@ -13,6 +19,7 @@
     <PackageVersion Include="JsonSchema.Net.Generation" Version="7.1.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.BannedApiAnalyzers" Version="4.14.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.3" />
+    <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.3" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.3" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.3" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="10.0.3" />
@@ -35,6 +42,7 @@
     <PackageVersion Include="StyleCop.Analyzers" Version="1.1.118" />
     <PackageVersion Include="System.CommandLine" Version="2.0.3" />
     <PackageVersion Include="System.Text.Json" Version="10.0.3" />
+    <PackageVersion Include="System.Threading.Channels" Version="10.0.3" />
     <PackageVersion Include="System.Threading.RateLimiting" Version="10.0.3" />
     <PackageVersion Include="Testcontainers" Version="4.10.0" />
     <PackageVersion Include="TngTech.ArchUnitNET.NUnit" Version="0.13.2" />

Drift.sln

@@ -73,6 +73,32 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Cli.Settings.SchemaGenerato
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Common.Schemas", "src\Common.Schemas\Common.Schemas.csproj", "{BAC0F9AF-CAE2-43FB-AF47-B9AC7B62544B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Networking.PeerStreaming.Grpc", "src\Networking.PeerStreaming.Grpc\Networking.PeerStreaming.Grpc.csproj", "{8ED3FF22-90D2-4F08-A079-55FE7127D1C7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Networking.Cluster", "src\Networking.Cluster\Networking.Cluster.csproj", "{091D3DCE-F062-4D40-A8F6-5B6F123ED713}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Networking.PeerStreaming.Core", "src\Networking.PeerStreaming.Core\Networking.PeerStreaming.Core.csproj", "{80445644-7342-4C6D-88E5-BF27126FE9A2}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Agent.Hosting", "src\Agent.Hosting\Agent.Hosting.csproj", "{655124DB-312F-4135-B104-20518CAFDA82}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Networking.PeerStreaming.Server", "src\Networking.PeerStreaming.Server\Networking.PeerStreaming.Server.csproj", "{A26B4527-6EBF-4A20-8E75-945CCD59016B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Networking.PeerStreaming.Client", "src\Networking.PeerStreaming.Client\Networking.PeerStreaming.Client.csproj", "{E69772D3-8A07-414F-8F9A-30370D81A972}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Networking", "Networking", "{75F8AA01-64B6-4EF6-A1B2-CC6E8745A2CC}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Networking.PeerStreaming.Core.Abstractions", "src\Networking.PeerStreaming.Core.Abstractions\Networking.PeerStreaming.Core.Abstractions.csproj", "{ED4522C5-C32B-4FDB-B1BA-82D40D1EC403}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Networking.PeerStreaming.Tests", "src\Networking.PeerStreaming.Tests\Networking.PeerStreaming.Tests.csproj", "{9DFDD692-22F8-4F9A-8808-94E318863D23}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Agent.PeerProtocol", "src\Agent.PeerProtocol\Agent.PeerProtocol.csproj", "{7C72C2AE-2888-47A0-AAA4-61CC66B9F941}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FeatureFlagsDELETE", "src\FeatureFlagsDELETE\FeatureFlagsDELETE.csproj", "{31593F51-0B46-4A2B-AA60-88A10D3195F3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FeatureFlagsDELETE.Tests", "src\FeatureFlagsDELETE.Tests\FeatureFlagsDELETE.Tests.csproj", "{24EA16E4-A73B-42F5-B315-FB9AE1B3F9AF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Agent.PeerProtocol.Tests", "src\Agent.PeerProtocol.Tests\Agent.PeerProtocol.Tests.csproj", "{C4576156-BD24-463F-88F2-8A4378855BCC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -177,11 +203,66 @@ Global
 		{BAC0F9AF-CAE2-43FB-AF47-B9AC7B62544B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{BAC0F9AF-CAE2-43FB-AF47-B9AC7B62544B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{BAC0F9AF-CAE2-43FB-AF47-B9AC7B62544B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8ED3FF22-90D2-4F08-A079-55FE7127D1C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8ED3FF22-90D2-4F08-A079-55FE7127D1C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8ED3FF22-90D2-4F08-A079-55FE7127D1C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8ED3FF22-90D2-4F08-A079-55FE7127D1C7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{091D3DCE-F062-4D40-A8F6-5B6F123ED713}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{091D3DCE-F062-4D40-A8F6-5B6F123ED713}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{091D3DCE-F062-4D40-A8F6-5B6F123ED713}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{091D3DCE-F062-4D40-A8F6-5B6F123ED713}.Release|Any CPU.Build.0 = Release|Any CPU
+		{80445644-7342-4C6D-88E5-BF27126FE9A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{80445644-7342-4C6D-88E5-BF27126FE9A2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{80445644-7342-4C6D-88E5-BF27126FE9A2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{80445644-7342-4C6D-88E5-BF27126FE9A2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{655124DB-312F-4135-B104-20518CAFDA82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{655124DB-312F-4135-B104-20518CAFDA82}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{655124DB-312F-4135-B104-20518CAFDA82}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{655124DB-312F-4135-B104-20518CAFDA82}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A26B4527-6EBF-4A20-8E75-945CCD59016B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A26B4527-6EBF-4A20-8E75-945CCD59016B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A26B4527-6EBF-4A20-8E75-945CCD59016B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A26B4527-6EBF-4A20-8E75-945CCD59016B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E69772D3-8A07-414F-8F9A-30370D81A972}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E69772D3-8A07-414F-8F9A-30370D81A972}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E69772D3-8A07-414F-8F9A-30370D81A972}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E69772D3-8A07-414F-8F9A-30370D81A972}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ED4522C5-C32B-4FDB-B1BA-82D40D1EC403}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ED4522C5-C32B-4FDB-B1BA-82D40D1EC403}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ED4522C5-C32B-4FDB-B1BA-82D40D1EC403}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ED4522C5-C32B-4FDB-B1BA-82D40D1EC403}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9DFDD692-22F8-4F9A-8808-94E318863D23}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9DFDD692-22F8-4F9A-8808-94E318863D23}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9DFDD692-22F8-4F9A-8808-94E318863D23}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9DFDD692-22F8-4F9A-8808-94E318863D23}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7C72C2AE-2888-47A0-AAA4-61CC66B9F941}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7C72C2AE-2888-47A0-AAA4-61CC66B9F941}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7C72C2AE-2888-47A0-AAA4-61CC66B9F941}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7C72C2AE-2888-47A0-AAA4-61CC66B9F941}.Release|Any CPU.Build.0 = Release|Any CPU
+		{31593F51-0B46-4A2B-AA60-88A10D3195F3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{31593F51-0B46-4A2B-AA60-88A10D3195F3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{31593F51-0B46-4A2B-AA60-88A10D3195F3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{31593F51-0B46-4A2B-AA60-88A10D3195F3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{24EA16E4-A73B-42F5-B315-FB9AE1B3F9AF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{24EA16E4-A73B-42F5-B315-FB9AE1B3F9AF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{24EA16E4-A73B-42F5-B315-FB9AE1B3F9AF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{24EA16E4-A73B-42F5-B315-FB9AE1B3F9AF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C4576156-BD24-463F-88F2-8A4378855BCC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C4576156-BD24-463F-88F2-8A4378855BCC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C4576156-BD24-463F-88F2-8A4378855BCC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C4576156-BD24-463F-88F2-8A4378855BCC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{8523E9E0-F412-41B7-B361-ADE639FFAF24} = {C0698EF0-61C8-403E-8E93-1F1D34C5B910}
 		{FEA2FBBE-785F-4187-8242-FD348F9E78AF} = {C0698EF0-61C8-403E-8E93-1F1D34C5B910}
 		{272166CF-E425-45F8-984F-FAFD3CE953C9} = {C0698EF0-61C8-403E-8E93-1F1D34C5B910}
 		{DD70FBC7-8367-45B3-8D3D-757F1CDF6531} = {C0698EF0-61C8-403E-8E93-1F1D34C5B910}
+		{091D3DCE-F062-4D40-A8F6-5B6F123ED713} = {75F8AA01-64B6-4EF6-A1B2-CC6E8745A2CC}
+		{8ED3FF22-90D2-4F08-A079-55FE7127D1C7} = {75F8AA01-64B6-4EF6-A1B2-CC6E8745A2CC}
+		{80445644-7342-4C6D-88E5-BF27126FE9A2} = {75F8AA01-64B6-4EF6-A1B2-CC6E8745A2CC}
+		{A26B4527-6EBF-4A20-8E75-945CCD59016B} = {75F8AA01-64B6-4EF6-A1B2-CC6E8745A2CC}
+		{E69772D3-8A07-414F-8F9A-30370D81A972} = {75F8AA01-64B6-4EF6-A1B2-CC6E8745A2CC}
+		{ED4522C5-C32B-4FDB-B1BA-82D40D1EC403} = {75F8AA01-64B6-4EF6-A1B2-CC6E8745A2CC}
+		{9DFDD692-22F8-4F9A-8808-94E318863D23} = {75F8AA01-64B6-4EF6-A1B2-CC6E8745A2CC}
 	EndGlobalSection
 EndGlobal

README_dev.md

@@ -35,6 +35,10 @@
   One or more addresses (MAC, IPv4, IPv6, and/or hostname) that together serve as a unique identifier for a network
   device.
 
+- **Agent**  
+  A running instance of Drift in agent mode that reports network state to other Drift peers.
+  Agents help ensure full network visibility by uncovering state that's only observable when scanning from specific subnets.
+
 ## Concepts
 
 ### Device ID

build-utils/Build.Utilities.Tests/Build.Utilities.Tests.csproj

@@ -1,11 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <ItemGroup>
-    <PackageReference Include="TUnit" />
+    <ProjectReference Include="..\Build.Utilities\Build.Utilities.csproj" />
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\Build.Utilities\Build.Utilities.csproj" />
+    <PackageReference Include="TUnit" />
   </ItemGroup>
 
 </Project>

build/_build.csproj

@@ -13,14 +13,15 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\build-utils\Build.Utilities\Build.Utilities.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="HLabs.Containers" />
     <PackageReference Include="HLabs.ImageReferences" />
     <PackageReference Include="Octokit" />
     <PackageReference Include="Semver" />
     <PackageReference Include="Nuke.Common" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\build-utils\Build.Utilities\Build.Utilities.csproj" />
-  </ItemGroup>
-
 </Project>

containerlab/blog.txt

@@ -0,0 +1,3 @@
+install
+
+create topology

containerlab/topo1.clab.yaml

@@ -0,0 +1,23 @@
+name: drift-topo1
+topology:
+  nodes:
+    switch|bp1:
+      kind: bridge
+      network-mode: container:bp1
+    bp1:
+      kind: linux
+      image: alpine:latest
+    pc1:
+      kind: linux
+      image: hojmark/drift
+      cmd: scan -i
+    pc2:
+      kind: linux
+      image: agent start --adoptable
+    pc3:
+      kind: linux
+      image: agent start --adoptable
+  links:
+    - endpoints: [ "pc1:eth1", "switch|bp1:eth1" ]
+    - endpoints: [ "pc2:eth1", "switch|bp1:eth2" ]
+    - endpoints: [ "pc3:eth1", "switch|bp1:eth3" ]

src/Agent.Hosting/Agent.Hosting.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Agent.PeerProtocol\Agent.PeerProtocol.csproj" />
+    <ProjectReference Include="..\Networking.Cluster\Networking.Cluster.csproj" />
+    <ProjectReference Include="..\Networking.PeerStreaming.Server\Networking.PeerStreaming.Server.csproj" />
+    <ProjectReference Include="..\Networking.PeerStreaming.Client\Networking.PeerStreaming.Client.csproj" />
+  </ItemGroup>
+
+</Project>

src/Agent.Hosting/AgentHost.cs

@@ -0,0 +1,70 @@
+using Drift.Agent.PeerProtocol.Subnets;
+using Drift.Networking.PeerStreaming.Client;
+using Drift.Networking.PeerStreaming.Core;
+using Drift.Networking.PeerStreaming.Server;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace Drift.Agent.Hosting;
+
+public static class AgentHost {
+  public static Task Run(
+    ushort port,
+    ILogger logger,
+    Action<IServiceCollection>? configureServices,
+    CancellationToken cancellationToken,
+    TaskCompletionSource? ready = null
+  ) {
+    var app = Build( port, logger, configureServices, ready );
+    return app.RunAsync( cancellationToken );
+  }
+
+  private static WebApplication Build(
+    ushort port,
+    ILogger logger,
+    Action<IServiceCollection>? configureServices = null,
+    TaskCompletionSource? ready = null
+  ) {
+    var builder = WebApplication.CreateSlimBuilder();
+
+    builder.Logging.ClearProviders();
+    builder.Services.AddSingleton( logger );
+    builder.Services.AddPeerStreamingServer( options => {
+      options.EnableDetailedErrors = true;
+    } );
+    builder.Services.AddPeerStreamingClient();
+    var peerStreamingOptions = new PeerStreamingOptions { MessageAssembly = typeof(SubnetsRequest).Assembly };
+    builder.Services.AddPeerStreamingCore( peerStreamingOptions );
+    configureServices?.Invoke( builder.Services );
+
+    builder.WebHost.ConfigureKestrel( options => {
+      options.ListenLocalhost( port, o => {
+        o.Protocols = HttpProtocols.Http2; // Allow HTTP/2 over plain HTTP i.e., non-HTTPS
+      } );
+    } );
+
+    var app = builder.Build();
+
+    peerStreamingOptions.StoppingToken = app.Lifetime.ApplicationStopping;
+
+    app.MapPeerStreamingServerEndpoints();
+
+    app.Lifetime.ApplicationStarted.Register( () => {
+      logger.LogInformation( "Listening for incoming connections on port {Port}", port );
+      logger.LogInformation( "Agent started" );
+      ready?.TrySetResult();
+    } );
+    app.Lifetime.ApplicationStopping.Register( () => {
+      logger.LogInformation( "Agent stopping..." );
+    } );
+    app.Lifetime.ApplicationStopped.Register( () => {
+      logger.LogInformation( "Agent stopped" );
+    } );
+
+    return app;
+  }
+}

src/Agent.PeerProtocol.Tests/Agent.PeerProtocol.Tests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <ItemGroup>
+    <Using Include="NUnit.Framework" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Agent.PeerProtocol\Agent.PeerProtocol.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="NUnit" />
+    <PackageReference Include="NUnit.Analyzers" />
+    <PackageReference Include="NUnit3TestAdapter" />
+    <PackageReference Include="Verify.NUnit" />
+  </ItemGroup>
+
+</Project>

src/Agent.PeerProtocol.Tests/AssemblyInfo.cs

@@ -0,0 +1 @@
+[assembly: Category( "Unit" )]