List detector

.github/actions/install-dependencies/action.yml

@@ -29,4 +29,4 @@ runs:
       if: ${{ inputs.clang-tools == 'true' }}
       shell: bash
       run: |
-        dnf install -y clang clang-tools-extra
+        dnf install -y clang clang-tools-extra

.github/workflows/ci.yml

@@ -32,6 +32,11 @@ jobs:
     uses: ./.github/workflows/build.yml
     with:
       os: ${{ matrix.os }}
+  tests:
+    needs: [build-os-matrix, build]
+    uses: ./.github/workflows/tests.yml
+    with:
+      os: "oraclelinux:9"
   rpm-install:
     needs: [build-os-matrix, build]
     strategy:

.github/workflows/tests.yml

@@ -0,0 +1,37 @@
+name: tests
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        required: true
+        type: string
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    container: ${{ inputs.os }}
+    steps:
+    - name: Install git
+      run: dnf install -y git
+    - name: Check out repository code
+      uses: actions/checkout@v4
+    - name: Install dependencies
+      uses: ./.github/actions/install-dependencies
+    - name: Install nemea
+      run: |
+          dnf copr enable @CESNET/NEMEA-testing
+          dnf copr enable @CESNET/NEMEA
+          dnf install -y epel-release
+          dnf install -y nemea-framework-devel
+          dnf install -y nemea
+          dnf install -y procps-ng autoconf
+          echo PATH=/usr/bin/nemea:$PATH >> $GITHUB_ENV
+    - name: Compile modules
+      run: |
+        cmake -S . -B build -DNM_NG_ENABLE_TESTS=On
+        make -C build install
+    - name: Run tests
+      run: |
+        echo "Path=$PATH"
+        make -C build tests

CMakeLists.txt

@@ -20,6 +20,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 option(NM_NG_ENABLE_DOC_DOXYGEN "Enable build of code documentation" OFF)
 option(NM_NG_BUILD_WITH_ASAN    "Build with Address Sanitizer (only for CMAKE_BUILD_TYPE=Debug)" OFF)
 option(NM_NG_BUILD_WITH_UBSAN   "Build with Undefined Behavior Sanitizer (only for CMAKE_BUILD_TYPE=Debug)" OFF)
+option(NM_NG_ENABLE_TESTS       "Build with tests of modules" OFF)
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic -Wall -Wextra -Wunused -Wconversion -Wsign-conversion")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -Werror")
@@ -41,3 +42,7 @@ add_subdirectory(modules)
 add_subdirectory(common)
 add_subdirectory(pkg)
 add_subdirectory(doc)
+
+if (NM_NG_ENABLE_TESTS)
+	include(cmake/tests.cmake)
+endif()

README.md

@@ -4,6 +4,7 @@ This repository contains basic modules of the [NEMEA
 system](https://github.com/CESNET/Nemea). The modules and their
 functionality/purposes are:
 
+* [ListDetector](modules/listdetector/): forwards records that match rules list.
 * [Sampler](modules/sampler/): sample records at the given rate.
 * [Telemetry](modules/telemetry/): provides unirec telemetry of the input interface.
 * [Deduplicator](modules/deduplicator/): omit duplicate records.

cmake/tests.cmake

@@ -0,0 +1,27 @@
+# File searches for test.sh script in each module subdirectory and runs it if present
+
+file(GLOB MODULE_DIRS RELATIVE ${CMAKE_SOURCE_DIR}/modules ${CMAKE_SOURCE_DIR}/modules/*)
+enable_testing()
+
+add_custom_target(tests
+	COMMAND ctest --output-on-failure
+	VERBATIM
+)
+
+foreach(MODULE ${MODULE_DIRS})
+	if (NOT IS_DIRECTORY ${CMAKE_SOURCE_DIR}/modules/${MODULE})
+		continue()
+	endif()
+	set(TEST_SCRIPT "${CMAKE_SOURCE_DIR}/modules/${MODULE}/tests/test.sh")
+
+	if (EXISTS ${TEST_SCRIPT})
+		add_test(NAME Test${MODULE} COMMAND bash ${TEST_SCRIPT} ${CMAKE_BINARY_DIR}/modules/${MODULE}/src/${MODULE})
+		add_dependencies(tests ${MODULE})
+	else()
+		add_custom_target(print_missing_test_for_${MODULE}
+			COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --yellow --bold "No test.sh found for: ${MODULE}. Skipping..."
+			VERBATIM
+		)
+		add_dependencies(tests print_missing_test_for_${MODULE})
+	endif()
+endforeach()

modules/CMakeLists.txt

@@ -1,3 +1,4 @@
+add_subdirectory(listDetector)
 add_subdirectory(sampler)
 add_subdirectory(telemetry)
 add_subdirectory(deduplicator)

modules/listDetector/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(src)

modules/listDetector/README.md

@@ -0,0 +1,84 @@
+# ListDetector module - README
+
+## Description
+The module analyzes Unirec records by comparing them against a set of predefined rules in a rule list.
+Rule list can be blacklist or whitelist. It identifies and forwards records that do not match to the whitelist rules or match blacklist rules.
+
+## Interfaces
+- Input: 1
+- Output: 1
+
+## Parameters
+### Common TRAP parameters
+- `-h [trap,1]`      Print help message for this module / for libtrap specific parameters.
+- `-i IFC_SPEC`      Specification of interface types and their parameters.
+- `-v`               Be verbose.
+- `-vv`              Be more verbose.
+- `-vvv`             Be even more verbose.
+
+### Module specific parameters
+- `-r, --rules <file>`  ListDetector module rules in CSV format
+- `-lm, --listmode <file>`  ListDetector mode - whitelist or blacklist
+- `-m, --appfs-mountpoint <path>` Path where the appFs directory will be mounted
+
+## CSV rules format
+The first row of CSV specifies the unirec types and names of fields that will be
+used for whitelisting or blacklisting.
+
+The supported unirec types are: `uint8`, `int8`, `uint16`, `int16`, `uint32`, `int32`,
+`uint64`, `int64`, `char`, `ipaddr` and `string`.
+
+- Empty values match everyting.
+
+- Numeric types match the exact value.
+
+- IP address (`ipaddr`) can be either ipv4 or ipv6 address.
+The ip address can optionally have a prefix.
+If there is no prefix, the address must match exactly.
+	- Examples: `127.0.0.1`, `127.0.0.0/24`
+
+- String match a regex pattern. Regex patterns support extended grep syntax.
+   - Examples: `R"(^www.google.com$)"`, `R"(.*google\.com$)"`
+
+### Example CSV file
+
+```
+ipaddr SRC_IP,uint16 DST_PORT,uint16 SRC_PORT
+10.0.0.1,443,53530
+10.0.0.2,443,53531
+```
+
+```
+ipaddr SCR_IP,string QUIC_SNI
+10.0.0.1/24,R"(.*google\.com$)"
+```
+
+## Usage Examples
+```
+# Data from the input unix socket interface "trap_in" is processed, and entries that
+do not match the defined rules in the "csvWhitelist.csv" file are forwarded to the
+output interface "trap_out."
+
+$ listDetector -i u:trap_in,u:trap_out -r csvWhitelist.csv
+```
+```
+# Data from the input unix socket interface "trap_in" is processed, and entries that
+match the defined rules in the "csvblacklist.csv" file are forwarded to the
+output interface "trap_out."
+
+$ listDetector -i u:trap_in,u:trap_out -lm bl -r csvBlacklist.csv
+```
+
+## Telemetry data format
+```
+├─ input/
+│  └─ stats
+└─ listDetector/
+   ├─ aggStats
+   └─ rules/
+      ├─ 0
+      ├─ 1
+      └ ...
+```
+
+Each rule has its own file named according to the order of the rules in the configuration file.

modules/listDetector/src/CMakeLists.txt

@@ -0,0 +1,26 @@
+add_executable(listDetector
+	main.cpp
+	configParser.cpp
+	csvConfigParser.cpp
+	ipAddressPrefix.cpp
+	rule.cpp
+	ruleBuilder.cpp
+	listDetector.cpp
+	ipAddressFieldMatcher.cpp
+	fieldsMatcher.cpp
+	rulesMatcher.cpp
+)
+
+target_link_libraries(listDetector PRIVATE
+	telemetry::telemetry
+	telemetry::appFs
+	common
+	rapidcsv
+	unirec::unirec++
+	unirec::unirec
+	trap::trap
+	argparse
+	xxhash
+)
+
+install(TARGETS listDetector DESTINATION ${INSTALL_DIR_BIN})

modules/listDetector/src/configParser.cpp

@@ -0,0 +1,90 @@
+/**
+ * @file
+ * @author Pavel Siska <siska@cesnet.cz>
+ * @brief Implemetation of the ConfigParser base class for parsing and processing list detector
+ * configuration data
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include "configParser.hpp"
+
+#include <numeric>
+#include <regex>
+#include <stdexcept>
+#include <string_view>
+
+namespace {
+
+std::string concatenateVectorOfStrings(
+	const std::vector<std::string>& vectorToConcatenate,
+	const std::string& delimiter = ",")
+{
+	if (vectorToConcatenate.empty()) {
+		return "";
+	}
+
+	std::string concatenatedString = std::accumulate(
+		vectorToConcatenate.begin() + 1,
+		vectorToConcatenate.end(),
+		vectorToConcatenate.front(),
+		[&](const std::string& accum, const std::string& str) { return accum + delimiter + str; });
+
+	return concatenatedString;
+}
+
+} // namespace
+
+namespace ListDetector {
+
+void ConfigParser::setUnirecTemplate(const std::vector<UnirecTypeName>& unirecTemplateDescription)
+{
+	m_unirecTemplateDescription = unirecTemplateDescription;
+}
+
+std::string ConfigParser::getUnirecTemplateDescription() const
+{
+	return concatenateVectorOfStrings(m_unirecTemplateDescription);
+}
+
+void ConfigParser::addRule(const RuleDescription& ruleDescription)
+{
+	m_rulesDescription.emplace_back(ruleDescription);
+}
+
+void ConfigParser::validate() const
+{
+	validateUnirecTemplate();
+	validateRules();
+}
+
+void ConfigParser::validateUnirecTemplate() const
+{
+	const std::regex unirecTemplateValidPattern(R"(^([^,\s]+ [^,\s]+,)*[^,\s]+ [^,\s]+$)");
+
+	const std::string unirecTemplateString
+		= concatenateVectorOfStrings(m_unirecTemplateDescription);
+	if (!std::regex_match(unirecTemplateString, unirecTemplateValidPattern)) {
+		m_logger->error("Unirec template header '{}' has invalid format.", unirecTemplateString);
+		throw std::invalid_argument("ConfigParser::validateUnirecTemplate() has failed");
+	}
+}
+
+void ConfigParser::validateRules() const
+{
+	for (const auto& ruleDescription : m_rulesDescription) {
+		if (ruleDescription.size() == m_unirecTemplateDescription.size()) {
+			continue;
+		}
+
+		m_logger->error(
+			"Rule '{}' has invalid number of columns. Expected {} columns, got {} "
+			"columns.",
+			concatenateVectorOfStrings(ruleDescription),
+			m_unirecTemplateDescription.size(),
+			ruleDescription.size());
+		throw std::invalid_argument("ConfigParser::validateRules() has failed");
+	}
+}
+
+} // namespace ListDetector