From 40bc236e007f9280d907288aa77d1afe9df350a2 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Tue, 16 Jan 2024 17:25:32 +0200 Subject: [PATCH 1/4] start adding NRedisStack --- .../Mbc.Log4Tc.Output.Redis.csproj | 16 ++++++ .../RedisLog4TcOutputFactory.cs | 22 ++++++++ .../RedisLog4TcOutputPlugin.cs | 14 +++++ .../Mbc.Log4Tc.Output.Redis/RedisOutput.cs | 51 +++++++++++++++++++ .../RedisOutputSettings.cs | 36 +++++++++++++ .../Mbc.Log4Tc.Service.csproj | 1 + source/Log4Tc/Mbc.Log4Tc.Service/Program.cs | 1 + 7 files changed, 141 insertions(+) create mode 100644 source/Log4Tc/Mbc.Log4Tc.Output.Redis/Mbc.Log4Tc.Output.Redis.csproj create mode 100644 source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisLog4TcOutputFactory.cs create mode 100644 source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisLog4TcOutputPlugin.cs create mode 100644 source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs create mode 100644 source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutputSettings.cs diff --git a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/Mbc.Log4Tc.Output.Redis.csproj b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/Mbc.Log4Tc.Output.Redis.csproj new file mode 100644 index 0000000..9801fc1 --- /dev/null +++ b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/Mbc.Log4Tc.Output.Redis.csproj @@ -0,0 +1,16 @@ + + + + netstandard2.0 + + + + + + + + + + + + diff --git a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisLog4TcOutputFactory.cs b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisLog4TcOutputFactory.cs new file mode 100644 index 0000000..9cc4ba1 --- /dev/null +++ b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisLog4TcOutputFactory.cs @@ -0,0 +1,22 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.ComponentModel.DataAnnotations; + +namespace Mbc.Log4Tc.Output.Redis +{ + public class RedisLog4TcOutputFactory : IOutputFactory + { + public string ShortTypeName => "redis"; + + public OutputHandlerBase Create(IServiceProvider serviceProvider, IConfigurationSection outputConfiguration) + { + var config = new RedisOutputSettings(); + + outputConfiguration.Bind(config); + Validator.ValidateObject(config, new ValidationContext(config)); + + return ActivatorUtilities.CreateInstance(serviceProvider, config); + } + } +} diff --git a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisLog4TcOutputPlugin.cs b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisLog4TcOutputPlugin.cs new file mode 100644 index 0000000..1a23300 --- /dev/null +++ b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisLog4TcOutputPlugin.cs @@ -0,0 +1,14 @@ +using Mbc.Log4Tc.Plugin; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace Mbc.Log4Tc.Output.Redis +{ + public class RedisLog4TcOutputPlugin : IPlugin + { + public void ConfigureServices(IServiceCollection services, IConfiguration configuration) + { + services.AddSingleton(); + } + } +} diff --git a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs new file mode 100644 index 0000000..b6d5d79 --- /dev/null +++ b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs @@ -0,0 +1,51 @@ +using Mbc.Log4Tc.Model; +using NRedisStack; +using System; +using System.Threading.Tasks; +using StackExchange.Redis; + +namespace Mbc.Log4Tc.Output.Redis +{ + /// + /// Outputs log entries to Redis. + /// + public class RedisOutput : OutputHandlerBase, IDisposable + { + private readonly ConfigurationOptions _configuration; // TODO: check if this should be configurationOptions + private IDatabase _redisClient; + + public RedisOutput(ConfigurationOptions configuration) + { + ConfigurationOptions configuration; + InitializeRedisClient(); + } + + public void Dispose() + { + _redisClient?.Dispose(); + _redisClient = null; + } + + protected override async Task ProcesLogEntriesAsync(IEnumerable logEntries) + { + var key = $"log:{logEntry.Logger}:{logEntry.Level}"; + var timestamp = new TimeStamp(logEntry.PlcTimestamp.ToUniversalTime().Ticks / TimeSpan.TicksPerMillisecond); + + var labels = new TimeSeriesLabel[] + { + new TimeSeriesLabel("Source", logEntry.Source), + new TimeSeriesLabel("Hostname", logEntry.Hostname), + // ... other labels like TaskName, AppName, etc. + }; + + var value = logEntry.FormattedMessage; // or any other numerical value relevant to your log + + await timeSeriesClient.AddAsync(key, timestamp, value, labels, duplicatePolicy: TsDuplicatePolicy.LAST); + } + } + + private void InitializeRedisClient() + { + _redisClient = ConnectionMultiplexer.Connect(_configuration); + } +} diff --git a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutputSettings.cs b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutputSettings.cs new file mode 100644 index 0000000..0165d8b --- /dev/null +++ b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutputSettings.cs @@ -0,0 +1,36 @@ +using System.ComponentModel.DataAnnotations; +using NRedisStack; + +namespace Mbc.Log4Tc.Output.Redis +{ + public class RedisOutputSettings + { + // TODO: Check which attributes should be here. + [Required] + [Url] + public string Url { get; set; } + + public string Username { get; set; } + + public string Password { get; set; } + + [Required] + public string Database { get; set; } + + public string RetentionPolicy { get; set; } + + public int WriteBatchSize { get; set; } = 20; + + public int WriteFlushIntervalMillis { get; set; } = 1000; + + public InfluxFormat Format { get; set; } = InfluxFormat.Arguments; + + public int SyslogFacilityCode { get; set; } = 16; + + public enum InfluxFormat + { + Arguments, + Syslog, + } + } +} diff --git a/source/Log4Tc/Mbc.Log4Tc.Service/Mbc.Log4Tc.Service.csproj b/source/Log4Tc/Mbc.Log4Tc.Service/Mbc.Log4Tc.Service.csproj index 1b28d7d..e6b3138 100644 --- a/source/Log4Tc/Mbc.Log4Tc.Service/Mbc.Log4Tc.Service.csproj +++ b/source/Log4Tc/Mbc.Log4Tc.Service/Mbc.Log4Tc.Service.csproj @@ -44,6 +44,7 @@ + diff --git a/source/Log4Tc/Mbc.Log4Tc.Service/Program.cs b/source/Log4Tc/Mbc.Log4Tc.Service/Program.cs index 2022422..f3c33ab 100644 --- a/source/Log4Tc/Mbc.Log4Tc.Service/Program.cs +++ b/source/Log4Tc/Mbc.Log4Tc.Service/Program.cs @@ -1,6 +1,7 @@ using Mbc.Log4Tc.Dispatcher; using Mbc.Log4Tc.Output.Graylog; using Mbc.Log4Tc.Output.InfluxDb; +// using Mbc.Log4Tc.Output.Redis; using Mbc.Log4Tc.Output.NLog; using Mbc.Log4Tc.Output.Sql; using Mbc.Log4Tc.Plugin; From bb742c1cfafbb86758ab5d72f005528e6f674dd9 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Thu, 18 Jan 2024 14:06:48 +0200 Subject: [PATCH 2/4] fix RedisOutput --- .../Mbc.Log4Tc.Output.Redis.Test.csproj | 17 ++++++++ .../AssemblyProperties.cs | 3 ++ .../Mbc.Log4Tc.Output.Redis/RedisOutput.cs | 42 +++++++++---------- source/Log4Tc/Mbc.Log4Tc.Service/Program.cs | 2 +- 4 files changed, 41 insertions(+), 23 deletions(-) create mode 100644 source/Log4Tc/Mbc.Log4Tc.Output.Redis.Test/Mbc.Log4Tc.Output.Redis.Test.csproj create mode 100644 source/Log4Tc/Mbc.Log4Tc.Output.Redis/AssemblyProperties.cs diff --git a/source/Log4Tc/Mbc.Log4Tc.Output.Redis.Test/Mbc.Log4Tc.Output.Redis.Test.csproj b/source/Log4Tc/Mbc.Log4Tc.Output.Redis.Test/Mbc.Log4Tc.Output.Redis.Test.csproj new file mode 100644 index 0000000..18dac2c --- /dev/null +++ b/source/Log4Tc/Mbc.Log4Tc.Output.Redis.Test/Mbc.Log4Tc.Output.Redis.Test.csproj @@ -0,0 +1,17 @@ + + + + net461 + + false + + + + + + + + + + + diff --git a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/AssemblyProperties.cs b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/AssemblyProperties.cs new file mode 100644 index 0000000..beea13c --- /dev/null +++ b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/AssemblyProperties.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Mbc.Log4Tc.Output.Redis.Test")] diff --git a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs index b6d5d79..2286d93 100644 --- a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs +++ b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs @@ -1,5 +1,8 @@ using Mbc.Log4Tc.Model; +using System.Collections.Generic; using NRedisStack; +using NRedisStack.RedisStackCommands; +using NRedisStack.DataTypes; using System; using System.Threading.Tasks; using StackExchange.Redis; @@ -11,41 +14,36 @@ namespace Mbc.Log4Tc.Output.Redis /// public class RedisOutput : OutputHandlerBase, IDisposable { - private readonly ConfigurationOptions _configuration; // TODO: check if this should be configurationOptions - private IDatabase _redisClient; + private readonly ConfigurationOptions _configuration; + private ConnectionMultiplexer? _redisConnection; + private IDatabase? _redisDb; + private TimeSeriesCommands _timeSeriesCommands; public RedisOutput(ConfigurationOptions configuration) { - ConfigurationOptions configuration; + _configuration = configuration; InitializeRedisClient(); } public void Dispose() { - _redisClient?.Dispose(); - _redisClient = null; + _redisConnection?.Close(); + _redisConnection = null; } - protected override async Task ProcesLogEntriesAsync(IEnumerable logEntries) + public override async Task ProcesLogEntriesAsync(IEnumerable logEntries) { - var key = $"log:{logEntry.Logger}:{logEntry.Level}"; - var timestamp = new TimeStamp(logEntry.PlcTimestamp.ToUniversalTime().Ticks / TimeSpan.TicksPerMillisecond); - - var labels = new TimeSeriesLabel[] + foreach (var logEntry in logEntries) { - new TimeSeriesLabel("Source", logEntry.Source), - new TimeSeriesLabel("Hostname", logEntry.Hostname), - // ... other labels like TaskName, AppName, etc. - }; - - var value = logEntry.FormattedMessage; // or any other numerical value relevant to your log - - await timeSeriesClient.AddAsync(key, timestamp, value, labels, duplicatePolicy: TsDuplicatePolicy.LAST); + // TODO: process log entry and write to redis time series. + } } - } - private void InitializeRedisClient() - { - _redisClient = ConnectionMultiplexer.Connect(_configuration); + private void InitializeRedisClient() + { + _redisConnection = ConnectionMultiplexer.Connect(_configuration); + _redisDb = _redisConnection.GetDatabase(); + _timeSeriesCommands = _redisDb.TS(); + } } } diff --git a/source/Log4Tc/Mbc.Log4Tc.Service/Program.cs b/source/Log4Tc/Mbc.Log4Tc.Service/Program.cs index f3c33ab..de40747 100644 --- a/source/Log4Tc/Mbc.Log4Tc.Service/Program.cs +++ b/source/Log4Tc/Mbc.Log4Tc.Service/Program.cs @@ -1,7 +1,7 @@ using Mbc.Log4Tc.Dispatcher; using Mbc.Log4Tc.Output.Graylog; using Mbc.Log4Tc.Output.InfluxDb; -// using Mbc.Log4Tc.Output.Redis; +using Mbc.Log4Tc.Output.Redis; using Mbc.Log4Tc.Output.NLog; using Mbc.Log4Tc.Output.Sql; using Mbc.Log4Tc.Plugin; From db31b514e68a1c014b874b9b96ab2640c76f6102 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Mon, 12 Feb 2024 14:58:52 +0200 Subject: [PATCH 3/4] docker compose --- .devcontainer/Dockerfile | 4 ++++ .devcontainer/devcontainer.json | 17 +++++++++++++++++ .devcontainer/docker-compose.yml | 24 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..6b65b9b --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,4 @@ +# Base development image on the .NET version used by Log4TC, adjust if different +FROM mcr.microsoft.com/devcontainers/dotnet:0-7.0-bullseye AS build + +# Add any additional setup required for Log4TC here diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..3ed5089 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +{ + "name": "Log4TC", + + "dockerComposeFile": "docker-compose.yml", + "service": "devcontainer", + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", + "customizations": { + "vscode": { + "extensions": [ + "ms-dotnettools.csharp", + "ms-dotnettools.csdevkit", + "ms-dotnettools.vscodeintellicode-csharp" + // Add any additional extensions specific to Log4TC here + ] + } + } +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 0000000..d7c424f --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,24 @@ +--- +version: "3.8" +services: + devcontainer: + build: + context: . + dockerfile: Dockerfile + volumes: + - ../..:/workspaces:cached + networks: + - redis + command: sleep infinity + environment: + REDIS: "redis-stack-7.2.0:6379" # default targeted Redis version + + redis-stack: + image: redis/redis-stack-server:7.2.0-RC3 + restart: unless-stopped + networks: + - redis + +networks: + # defines shared network for communicating with Redis + redis: From 5494aedc6c84e746b238558b876be57b458a42a4 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Mon, 12 Feb 2024 14:59:24 +0200 Subject: [PATCH 4/4] Redis output --- .../Mbc.Log4Tc.Output.Redis/RedisOutput.cs | 36 ++++++++++++++----- .../RedisOutputSettings.cs | 26 -------------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs index 2286d93..e19e4cd 100644 --- a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs +++ b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutput.cs @@ -1,11 +1,12 @@ using Mbc.Log4Tc.Model; using System.Collections.Generic; -using NRedisStack; -using NRedisStack.RedisStackCommands; -using NRedisStack.DataTypes; using System; using System.Threading.Tasks; using StackExchange.Redis; +using NRedisStack; +using NRedisStack.RedisStackCommands; +using NRedisStack.DataTypes; + namespace Mbc.Log4Tc.Output.Redis { @@ -27,23 +28,42 @@ public RedisOutput(ConfigurationOptions configuration) public void Dispose() { - _redisConnection?.Close(); + // _redisConnection?.Close(); _redisConnection = null; } public override async Task ProcesLogEntriesAsync(IEnumerable logEntries) { + ConnectionMultiplexer redis = ConnectionMultiplexer.Connect("localhost"); + IDatabase db = redis.GetDatabase(); + var ts = db.TS(); + // ts.Create(logEntries.First().Logger); + TimeSeriesLabel[] labels; foreach (var logEntry in logEntries) { - // TODO: process log entry and write to redis time series. + labels = + new TimeSeriesLabel[] + { + new TimeSeriesLabel("source", logEntry.Source ?? string.Empty), + new TimeSeriesLabel("hostname", logEntry.Hostname ?? string.Empty), + new TimeSeriesLabel("logger", logEntry.Logger ?? string.Empty), + new TimeSeriesLabel("level", logEntry.Level.ToString()), + new TimeSeriesLabel("appName", logEntry.AppName ?? string.Empty), + new TimeSeriesLabel("projectName", logEntry.ProjectName ?? string.Empty), + new TimeSeriesLabel("taskName", logEntry.TaskName ?? string.Empty), + new TimeSeriesLabel("taskIndex", logEntry.TaskIndex.ToString()), + new TimeSeriesLabel("taskCycleCounter", logEntry.TaskCycleCounter.ToString()), + new TimeSeriesLabel("onlineChangeCount", logEntry.OnlineChangeCount.ToString()), + }; + await ts.AddAsync(logEntry.Logger, logEntry.PlcTimestamp, 1.1, labels: labels); } } private void InitializeRedisClient() { - _redisConnection = ConnectionMultiplexer.Connect(_configuration); - _redisDb = _redisConnection.GetDatabase(); - _timeSeriesCommands = _redisDb.TS(); + this._redisConnection = ConnectionMultiplexer.Connect(this._configuration); + this._redisDb = this._redisConnection.GetDatabase(); + this._timeSeriesCommands = this._redisDb.TS(); } } } diff --git a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutputSettings.cs b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutputSettings.cs index 0165d8b..b1ca39a 100644 --- a/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutputSettings.cs +++ b/source/Log4Tc/Mbc.Log4Tc.Output.Redis/RedisOutputSettings.cs @@ -6,31 +6,5 @@ namespace Mbc.Log4Tc.Output.Redis public class RedisOutputSettings { // TODO: Check which attributes should be here. - [Required] - [Url] - public string Url { get; set; } - - public string Username { get; set; } - - public string Password { get; set; } - - [Required] - public string Database { get; set; } - - public string RetentionPolicy { get; set; } - - public int WriteBatchSize { get; set; } = 20; - - public int WriteFlushIntervalMillis { get; set; } = 1000; - - public InfluxFormat Format { get; set; } = InfluxFormat.Arguments; - - public int SyslogFacilityCode { get; set; } = 16; - - public enum InfluxFormat - { - Arguments, - Syslog, - } } }