From 371eddd78205cf3c4ebc726983874bb2f8678924 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Thu, 12 Sep 2024 16:33:37 +0200 Subject: [PATCH 01/15] ListDetector - Introduce ConfigParser class --- modules/listDetector/src/configParser.cpp | 90 +++++++++++++++++++++++ modules/listDetector/src/configParser.hpp | 87 ++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 modules/listDetector/src/configParser.cpp create mode 100644 modules/listDetector/src/configParser.hpp diff --git a/modules/listDetector/src/configParser.cpp b/modules/listDetector/src/configParser.cpp new file mode 100644 index 00000000..6dae6b9e --- /dev/null +++ b/modules/listDetector/src/configParser.cpp @@ -0,0 +1,90 @@ +/** + * @file + * @author Pavel Siska + * @brief Implemetation of the ConfigParser base class for parsing and processing list detector + * configuration data + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "configParser.hpp" + +#include +#include +#include +#include + +namespace { + +std::string concatenateVectorOfStrings( + const std::vector& vectorToConcatenate, + const std::string& delimiter = ",") +{ + if (vectorToConcatenate.empty()) { + return ""; + } + + std::string concatenatedString = std::accumulate( + vectorToConcatenate.begin() + 1, + vectorToConcatenate.end(), + vectorToConcatenate.front(), + [&](const std::string& accum, const std::string& str) { return accum + delimiter + str; }); + + return concatenatedString; +} + +} // namespace + +namespace ListDetector { + +void ConfigParser::setUnirecTemplate(const std::vector& unirecTemplateDescription) +{ + m_unirecTemplateDescription = unirecTemplateDescription; +} + +std::string ConfigParser::getUnirecTemplateDescription() const +{ + return concatenateVectorOfStrings(m_unirecTemplateDescription); +} + +void ConfigParser::addRule(const RuleDescription& ruleDescription) +{ + m_rulesDescription.emplace_back(ruleDescription); +} + +void ConfigParser::validate() const +{ + validateUnirecTemplate(); + validateRules(); +} + +void ConfigParser::validateUnirecTemplate() const +{ + const std::regex unirecTemplateValidPattern(R"(^([^,\s]+ [^,\s]+,)*[^,\s]+ [^,\s]+$)"); + + const std::string unirecTemplateString + = concatenateVectorOfStrings(m_unirecTemplateDescription); + if (!std::regex_match(unirecTemplateString, unirecTemplateValidPattern)) { + m_logger->error("Unirec template header '{}' has invalid format.", unirecTemplateString); + throw std::invalid_argument("ConfigParser::validateUnirecTemplate() has failed"); + } +} + +void ConfigParser::validateRules() const +{ + for (const auto& ruleDescription : m_rulesDescription) { + if (ruleDescription.size() == m_unirecTemplateDescription.size()) { + continue; + } + + m_logger->error( + "Rule '{}' has invalid number of columns. Expected {} columns, got {} " + "columns.", + concatenateVectorOfStrings(ruleDescription), + m_unirecTemplateDescription.size(), + ruleDescription.size()); + throw std::invalid_argument("ConfigParser::validateRules() has failed"); + } +} + +} // namespace ListDetector diff --git a/modules/listDetector/src/configParser.hpp b/modules/listDetector/src/configParser.hpp new file mode 100644 index 00000000..ee764110 --- /dev/null +++ b/modules/listDetector/src/configParser.hpp @@ -0,0 +1,87 @@ +/** + * @file + * @author Pavel Siska + * @brief Declaration of the ConfigParser base class for parsing and processing list detector + * configuration data + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "logger/logger.hpp" + +#include +#include + +namespace ListDetector { + +/** + * @brief Base class for parsing and processing list detector configuration data. + * + * The `ConfigParser` class provides functionality for parsing and processing list detector + * configuration data. It serves as a base class for specific parsers, such as CSV parsers, and + * offers methods for setting the Unirec template, adding rules, and performing + * validation. + */ +class ConfigParser { +public: + using UnirecTypeName = std::string; ///< Type to keep Unirec type + using TypeNameValue = std::string; ///< Type of rule field value + using RuleDescription = std::vector; ///< Type of rule description + + virtual ~ConfigParser() noexcept = default; + + /** + * Get the Unirec template description in the following format + * + * Example format: "uint32 FOO,uint8 BAR,float FOO2" + * + * @return A string representing the Unirec template. + */ + std::string getUnirecTemplateDescription() const; + + /** + * Get the list of rules descriptions. + * + * This method returns a vector containing the descriptions of rules as a list of + * TypeNameValue vectors. + * + * @return A vector of TypeNameValue vectors representing rules descriptions. + */ + std::vector getRulesDescription() const { return m_rulesDescription; } + +protected: + /** + * Set the Unirec template. + * + * @param unirecTemplate A vector of Unirec type names. + */ + void setUnirecTemplate(const std::vector& unirecTemplateDescription); + + /** + * Add a rule description to the configuration. + * + * @param ruleDescription A vector representing a rules list. + * + * @note The size of the vector must be equal to the size of the Unirec template. + * @note The order of the vector elements must correspond to the order of the Unirec template + */ + void addRule(const RuleDescription& ruleDescription); + + /** + * Perform validation of the configuration data. + */ + void validate() const; + +private: + void validateUnirecTemplate() const; + void validateRules() const; + + std::vector m_unirecTemplateDescription; + std::vector m_rulesDescription; + + std::shared_ptr m_logger = Nm::loggerGet("ConfigParser"); +}; + +} // namespace ListDetector From 831b8d7fffafd31af901d7dadb1b77c335a9caf8 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Thu, 12 Sep 2024 16:33:55 +0200 Subject: [PATCH 02/15] ListDetector - Introduce CsvConfigParser class --- modules/listDetector/src/csvConfigParser.cpp | 78 ++++++++++++++++++++ modules/listDetector/src/csvConfigParser.hpp | 45 +++++++++++ 2 files changed, 123 insertions(+) create mode 100644 modules/listDetector/src/csvConfigParser.cpp create mode 100644 modules/listDetector/src/csvConfigParser.hpp diff --git a/modules/listDetector/src/csvConfigParser.cpp b/modules/listDetector/src/csvConfigParser.cpp new file mode 100644 index 00000000..75415dfa --- /dev/null +++ b/modules/listDetector/src/csvConfigParser.cpp @@ -0,0 +1,78 @@ +/** + * @file + * @author Pavel Siska + * @brief Implementation of the CsvConfigParser class for parsing and processing CSV rule + * configuration file + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "csvConfigParser.hpp" + +#include +#include + +namespace ListDetector { + +static rapidcsv::LineReaderParams buildLineReaderParams() +{ + const bool skipCommentLines = true; + const char commentPrefix = '#'; + const bool skipEmptyLines = true; + + return rapidcsv::LineReaderParams(skipCommentLines, commentPrefix, skipEmptyLines); +} + +static rapidcsv::SeparatorParams buildSeparatorParams() +{ + const char separator = ','; + const bool trim = true; + + return rapidcsv::SeparatorParams(separator, trim); +} + +CsvConfigParser::CsvConfigParser(const std::string& configFilename) +{ + try { + loadFileAsDocument(configFilename); + parse(); + validate(); + } catch (const std::exception& ex) { + m_logger->error(ex.what()); + throw std::runtime_error("CsvConfigParser::CsvConfigParser() has failed"); + } +} + +void CsvConfigParser::loadFileAsDocument(const std::string& configFilename) +{ + auto lineReaderParams = buildLineReaderParams(); + auto separatorParams = buildSeparatorParams(); + + m_csvDocument.Load( + configFilename, + rapidcsv::LabelParams(), + separatorParams, + rapidcsv::ConverterParams(), + lineReaderParams); +} + +void CsvConfigParser::parse() +{ + parseHeader(); + parseRows(); +} + +void CsvConfigParser::parseHeader() +{ + setUnirecTemplate(m_csvDocument.GetColumnNames()); +} + +void CsvConfigParser::parseRows() +{ + for (size_t rowIndex = 0; rowIndex < m_csvDocument.GetRowCount(); rowIndex++) { + auto ruleDescription = m_csvDocument.GetRow(rowIndex); + addRule(ruleDescription); + } +} + +} // namespace ListDetector diff --git a/modules/listDetector/src/csvConfigParser.hpp b/modules/listDetector/src/csvConfigParser.hpp new file mode 100644 index 00000000..7d38d638 --- /dev/null +++ b/modules/listDetector/src/csvConfigParser.hpp @@ -0,0 +1,45 @@ +/** + * @file + * @author Pavel Siska + * @brief Declaration of the CsvConfigParser class for parsing and processing CSV rules file + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "configParser.hpp" +#include "logger/logger.hpp" + +#include +#include +#include + +namespace ListDetector { + +/** + * @brief Class for parsing and processing CSV rules file + */ +class CsvConfigParser : public ConfigParser { +public: + /** + * @brief Open and parse a CSV configuration file + * + * @param configFilename Path to configuration file + * @throw std::runtime_error If an error occurs during parsing + */ + explicit CsvConfigParser(const std::string& configFilename); + +private: + void loadFileAsDocument(const std::string& configFilename); + + void parse(); + void parseHeader(); + void parseRows(); + + rapidcsv::Document m_csvDocument; + + std::shared_ptr m_logger = Nm::loggerGet("CsvConfigParser"); +}; + +} // namespace ListDetector From 8c2b538591a8e63df44dfaed76f53a91be732e1a Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 11 Aug 2024 14:58:58 +0500 Subject: [PATCH 03/15] ListDetector - Introduce IpAddressPrefix class --- modules/listDetector/src/ipAddressPrefix.cpp | 83 ++++++++++++++++++++ modules/listDetector/src/ipAddressPrefix.hpp | 56 +++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 modules/listDetector/src/ipAddressPrefix.cpp create mode 100644 modules/listDetector/src/ipAddressPrefix.hpp diff --git a/modules/listDetector/src/ipAddressPrefix.cpp b/modules/listDetector/src/ipAddressPrefix.cpp new file mode 100644 index 00000000..c08526dd --- /dev/null +++ b/modules/listDetector/src/ipAddressPrefix.cpp @@ -0,0 +1,83 @@ +/** + * @file + * @author Pavel Siska + * @brief Implementation of the IpAddressPrefix class for IP address matching. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "ipAddressPrefix.hpp" + +#include +#include +#include +#include +#include +namespace ListDetector { + +static void validatePrefixLength(size_t prefix, size_t maxPrefix) +{ + if (prefix > maxPrefix) { + throw std::invalid_argument( + "Address prefix is too long. Given: " + std::to_string(prefix) + + ", max: " + std::to_string(maxPrefix)); + } +} + +IpAddressPrefix::IpAddressPrefix(Nemea::IpAddress ipAddress, size_t prefix) +{ + if (ipAddress.isIpv4()) { + validatePrefixLength(prefix, IPV4_MAX_PREFIX); + + if (prefix == 0) { + m_mask.ip = ip_from_int(0); + } else { + const size_t shift = IPV4_MAX_PREFIX - prefix; + m_mask.ip = ip_from_int(std::numeric_limits::max() << shift); + } + } else { + validatePrefixLength(prefix, IPV6_MAX_PREFIX); + + static const std::array emptyIp = {0}; + m_mask.ip = ip_from_16_bytes_be(emptyIp.data()); + + const size_t prefixBytes = prefix / 8; + const size_t prefixBits = prefix % 8; + + for (size_t bytesIndex = 0; bytesIndex < prefixBytes; bytesIndex++) { + m_mask.ip.bytes[bytesIndex] = UINT8_MAX; + } + + if (prefixBits != 0U) { + m_mask.ip.bytes[prefixBytes] = (uint8_t)(UINT8_MAX << (CHAR_BIT - prefixBits)); + } + } + + m_address = ipAddress & m_mask; +} + +bool IpAddressPrefix::isBelong(const Nemea::IpAddress& ipAddress) const noexcept +{ + return (ipAddress & m_mask) == m_address; +} + +std::pair, std::vector> +IpAddressPrefix::getIpAndMask() const noexcept +{ + std::vector ipAddress; + std::vector mask; + if (m_address.isIpv4()) { + for (auto i = 0; i < 4; i++) { + ipAddress.push_back((std::byte) ip_get_v4_as_bytes(&m_address.ip)[i]); + mask.push_back((std::byte) ip_get_v4_as_bytes(&m_mask.ip)[i]); + } + } else { + for (auto i = 0; i < 16; i++) { + ipAddress.push_back((std::byte) m_address.ip.bytes[i]); + mask.push_back((std::byte) m_mask.ip.bytes[i]); + } + } + return std::make_pair(ipAddress, mask); +} + +} // namespace ListDetector diff --git a/modules/listDetector/src/ipAddressPrefix.hpp b/modules/listDetector/src/ipAddressPrefix.hpp new file mode 100644 index 00000000..607bce1f --- /dev/null +++ b/modules/listDetector/src/ipAddressPrefix.hpp @@ -0,0 +1,56 @@ +/** + * @file + * @author Pavel Siska + * @brief Declaration of the IpAddressPrefix class for IP address matching. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include + +namespace ListDetector { + +/** + * @brief Represents an IP address with a specified prefix. + */ +class IpAddressPrefix { +public: + /** + * @brief Maximum prefix length for IPv4 addresses. + */ + static const size_t IPV4_MAX_PREFIX = 32; + + /** + * @brief Maximum prefix length for IPv6 addresses. + */ + static const size_t IPV6_MAX_PREFIX = 128; + + /** + * @brief Constructor for the IpAddressPrefix class. + * @param ipAddress The IP address. + * @param prefix The prefix length. + */ + IpAddressPrefix(Nemea::IpAddress ipAddress, size_t prefix); + + /** + * @brief Checks if a given IP address belongs to the same prefix. + * @param ipAddress The IP address to check. + * @return True if the IP address belongs to the same prefix, false otherwise. + */ + bool isBelong(const Nemea::IpAddress& ipAddress) const noexcept; + + /** + * @brief Returns prefix as network IP and mask. + * @return Pair of IP and mask as vectors of octets. + */ + std::pair, std::vector> getIpAndMask() const noexcept; + +private: + Nemea::IpAddress m_address; + Nemea::IpAddress m_mask; +}; + +} // namespace ListDetector From 3b892b47bba93f04a047f5d1075f6f8de83cf722 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 10 Nov 2024 05:00:29 +0100 Subject: [PATCH 04/15] List Detector - Introduce RuleBuilder class --- modules/listDetector/src/ruleBuilder.cpp | 177 +++++++++++++++++++++++ modules/listDetector/src/ruleBuilder.hpp | 63 ++++++++ 2 files changed, 240 insertions(+) create mode 100644 modules/listDetector/src/ruleBuilder.cpp create mode 100644 modules/listDetector/src/ruleBuilder.hpp diff --git a/modules/listDetector/src/ruleBuilder.cpp b/modules/listDetector/src/ruleBuilder.cpp new file mode 100644 index 00000000..956b2c92 --- /dev/null +++ b/modules/listDetector/src/ruleBuilder.cpp @@ -0,0 +1,177 @@ +/** + * @file + * @author Pavel Siska + * @brief Defines the RuleBuilder class for constructing rules. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "ruleBuilder.hpp" + +#include +#include +#include +#include +#include + +namespace ListDetector { + +template +static std::optional convertStringToType(const std::string& str) +{ + if (str.empty()) { + return std::nullopt; + } + + T typeValue; + if (std::from_chars(str.data(), str.data() + str.size(), typeValue).ec == std::errc {}) { + return typeValue; + } + + throw std::runtime_error("convertStringToType() has failed"); +} + +static std::optional convertStringToIpAddressPrefix(const std::string& ipStr) +{ + if (ipStr.empty()) { + return std::nullopt; + } + + const std::string delimiter = "/"; + const size_t delimiterPosition = ipStr.find_first_of(delimiter); + std::string prefixPart; + std::string ipAddressPart; + if (delimiterPosition == std::string::npos) { + ipAddressPart = ipStr; + } else { + ipAddressPart = ipStr.substr(0, delimiterPosition); + prefixPart = ipStr.substr(delimiterPosition + 1); + } + + const Nemea::IpAddress ipAddress(ipAddressPart); + size_t prefixNumber; + if (prefixPart.empty()) { + if (ipAddress.isIpv4()) { + prefixNumber = IpAddressPrefix::IPV4_MAX_PREFIX; + } else { + prefixNumber = IpAddressPrefix::IPV6_MAX_PREFIX; + } + } else { + if (std::from_chars(prefixPart.data(), prefixPart.data() + prefixPart.size(), prefixNumber) + .ec + != std::errc {}) { + throw std::runtime_error("convertStringToIpAddressPrefix() has failed"); + } + } + return IpAddressPrefix(ipAddress, prefixNumber); +} + +RuleBuilder::RuleBuilder(const std::string& unirecTemplateDescription) +{ + extractUnirecFieldsId(unirecTemplateDescription); + m_ipAddressFieldMatchers + = std::make_shared>(); +} + +void RuleBuilder::extractUnirecFieldsId(const std::string& unirecTemplateDescription) +{ + std::istringstream sstream(unirecTemplateDescription); + std::string token; + const char delimiter = ','; + + while (std::getline(sstream, token, delimiter)) { + const size_t pos = token.find(' '); + if (pos != std::string::npos) { + const std::string fieldName = token.substr(pos + 1); + const int fieldId = ur_get_id_by_name(fieldName.c_str()); + validateUnirecFieldId(fieldName, fieldId); + m_unirecFieldsId.emplace_back(fieldId); + } + } +} + +void RuleBuilder::validateUnirecFieldId(const std::string& fieldName, int unirecFieldId) +{ + if (unirecFieldId == UR_E_INVALID_NAME) { + m_logger->error("Invalid unirec field name '{}' in unirec template", fieldName); + throw std::runtime_error("RuleBuilder::validateUnirecFieldId() has failed"); + } +} + +Rule RuleBuilder::build(const ConfigParser::RuleDescription& ruleDescription) +{ + std::vector ruleFields; + + for (const auto& fieldValue : ruleDescription) { + auto ruleField = createRuleField(fieldValue, m_unirecFieldsId.at(ruleFields.size())); + ruleFields.emplace_back(ruleField); + } + + return Rule {ruleFields}; +} + +RuleField RuleBuilder::createRuleField(const std::string& fieldValue, ur_field_id_t fieldId) +{ + const ur_field_type_t unirecFieldType = ur_get_type(fieldId); + validateUnirecFieldType(fieldValue, unirecFieldType); + + switch (unirecFieldType) { + case UR_TYPE_STRING: { + auto fieldLength = fieldValue.length(); + constexpr const std::string_view regexStringPattern = "R\"()\""; + if (fieldLength >= regexStringPattern.length() && fieldValue[0] == 'R' + && fieldValue[1] == '\"' && fieldValue[2] == '(' && fieldValue[fieldLength - 2] == ')' + && fieldValue[fieldLength - 1] == '\"') { + auto stringValue = fieldValue.substr(3, fieldLength - regexStringPattern.length()); + return std::make_pair(fieldId, std::regex(stringValue, std::regex::egrep)); + } + return std::make_pair(fieldId, fieldValue); + } + case UR_TYPE_CHAR: + return std::make_pair(fieldId, convertStringToType(fieldValue)); + case UR_TYPE_UINT8: + return std::make_pair(fieldId, convertStringToType(fieldValue)); + case UR_TYPE_INT8: + return std::make_pair(fieldId, convertStringToType(fieldValue)); + case UR_TYPE_UINT16: + return std::make_pair(fieldId, convertStringToType(fieldValue)); + case UR_TYPE_INT16: + return std::make_pair(fieldId, convertStringToType(fieldValue)); + case UR_TYPE_UINT32: + return std::make_pair(fieldId, convertStringToType(fieldValue)); + case UR_TYPE_INT32: + return std::make_pair(fieldId, convertStringToType(fieldValue)); + case UR_TYPE_UINT64: + return std::make_pair(fieldId, convertStringToType(fieldValue)); + case UR_TYPE_INT64: + return std::make_pair(fieldId, convertStringToType(fieldValue)); + case UR_TYPE_IP: { + auto ruleField = std::make_pair(fieldId, convertStringToIpAddressPrefix(fieldValue)); + if (ruleField.second.has_value()) { + (*m_ipAddressFieldMatchers)[fieldId].addPrefix(ruleField.second.value()); + } else { + (*m_ipAddressFieldMatchers)[fieldId].addEmptyPrefix(); + } + return ruleField; + } + default: + m_logger->error("Unsopported unirec data type for field '{}'", ur_get_name(fieldId)); + throw std::runtime_error("RuleBuilder::createRuleField has failed"); + } +} + +void RuleBuilder::validateUnirecFieldType(const std::string& fieldTypeString, int unirecFieldType) +{ + if (unirecFieldType == UR_E_INVALID_TYPE) { + m_logger->error("Invalid unirec field type '{}' in unirec template", fieldTypeString); + throw std::runtime_error("RuleBuilder::validateUnirecFieldType() has failed"); + } +} + +std::shared_ptr> +RuleBuilder::getIpAddressFieldMatchers() const noexcept +{ + return m_ipAddressFieldMatchers; +} + +} // namespace ListDetector diff --git a/modules/listDetector/src/ruleBuilder.hpp b/modules/listDetector/src/ruleBuilder.hpp new file mode 100644 index 00000000..a6a8c627 --- /dev/null +++ b/modules/listDetector/src/ruleBuilder.hpp @@ -0,0 +1,63 @@ +/** + * @file + * @author Pavel Siska + * @author Damir Zainullin + * @brief Declares the RuleBuilder class for constructing rules. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "configParser.hpp" +#include "logger/logger.hpp" +#include "rule.hpp" + +#include +#include +#include +#include + +namespace ListDetector { + +/** + * @brief A class for building a Rule. + */ +class RuleBuilder { +public: + /** + * @brief Constructs a RuleBuilder with the specified Unirec template description. + * @param unirecTemplateDescription The description of the Unirec template. + */ + explicit RuleBuilder(const std::string& unirecTemplateDescription); + + /** + * @brief Builds a Rule based on the given rule description. + * @param ruleDescription The description of the rule. + * @return Constructed Rule. + */ + Rule build(const ConfigParser::RuleDescription& ruleDescription); + + /** + * @brief Getter for IP address field matchers. + * @return Shared pointer to unordered map of IP address field matcher, where id of Unirec field + * is a key. + */ + std::shared_ptr> + getIpAddressFieldMatchers() const noexcept; + +private: + void extractUnirecFieldsId(const std::string& unirecTemplateDescription); + void validateUnirecFieldId(const std::string& fieldName, int unirecFieldId); + void validateUnirecFieldType(const std::string& fieldTypeString, int unirecFieldType); + RuleField createRuleField(const std::string& fieldValue, ur_field_id_t fieldId); + + std::vector m_unirecFieldsId; + + std::shared_ptr m_logger = Nm::loggerGet("RuleBuilder"); + + std::shared_ptr> + m_ipAddressFieldMatchers; +}; + +} // namespace ListDetector From 5c97369762b84ce444fb8503b63fcf81f418c746 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 11 Aug 2024 15:00:27 +0500 Subject: [PATCH 05/15] ListDetector - Introduce Rule class --- modules/listDetector/src/rule.cpp | 118 +++++++++++++++++++++++++ modules/listDetector/src/rule.hpp | 137 ++++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 modules/listDetector/src/rule.cpp create mode 100644 modules/listDetector/src/rule.hpp diff --git a/modules/listDetector/src/rule.cpp b/modules/listDetector/src/rule.cpp new file mode 100644 index 00000000..0aaf67d7 --- /dev/null +++ b/modules/listDetector/src/rule.cpp @@ -0,0 +1,118 @@ +/** + * @file + * @author Pavel Siska + * @author Daniel Pelanek + * @brief Implementation of the Rule class. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "rule.hpp" + +#include +#include + +namespace ListDetector { + +static bool isDynamicRuleFieldMatched( + const RuleField& ruleField, + const Nemea::UnirecRecordView& unirecRecordView) +{ + const auto& [unirecFieldId, fieldPattern] = ruleField; + + if (!fieldPattern.has_value()) { + return false; + } + + switch (ur_get_type(unirecFieldId)) { + case UR_TYPE_STRING: { + if (std::holds_alternative(fieldPattern.value())) { + return true; + } + return std::regex_search( + unirecRecordView.getFieldAsType(unirecFieldId).begin(), + unirecRecordView.getFieldAsType(unirecFieldId).end(), + std::get(fieldPattern.value())); + } + case UR_TYPE_IP: + return true; + default: + throw std::runtime_error("Not a dynamic field"); + } + return false; +} + +Rule::Rule(std::vector ruleFields) + : M_RULE_FIELDS(std::move(ruleFields)) +{ +} + +bool Rule::isStringRuleField(const RuleField& ruleField) noexcept +{ + auto type = ur_get_type(ruleField.first); + return type == UR_TYPE_STRING; +} + +bool Rule::isStaticRuleField(const RuleField& ruleField) noexcept +{ + auto type = ur_get_type(ruleField.first); + return type != UR_TYPE_IP && type != UR_TYPE_STRING; +} + +bool Rule::dynamicFieldsMatch(const Nemea::UnirecRecordView& unirecRecordView) +{ + auto lambdaPredicate = [&](const auto& ruleField) { + return isStaticRuleField(ruleField) + || isDynamicRuleFieldMatched(ruleField, unirecRecordView) + || isWildcardRuleField(ruleField); + }; + + const bool isMatched = std::all_of(M_RULE_FIELDS.begin(), M_RULE_FIELDS.end(), lambdaPredicate); + if (isMatched) { + m_stats.matchedCount++; + } + return isMatched; +} + +const RuleStats& Rule::getStats() const noexcept +{ + return m_stats; +} + +const std::vector& Rule::getRuleFields() const noexcept +{ + return M_RULE_FIELDS; +} + +bool Rule::isWildcardRuleField(const RuleField& ruleField) noexcept +{ + return !ruleField.second.has_value(); +} + +bool Rule::isIPRuleField(const RuleField& ruleField) noexcept +{ + auto type = ur_get_type(ruleField.first); + return type == UR_TYPE_IP; +} + +bool Rule::isRegexRuleField(const RuleField& ruleField) noexcept +{ + return ruleField.second.has_value() + && std::holds_alternative(ruleField.second.value()); +} + +std::vector Rule::getPresentedStaticFieldsMask() const noexcept +{ + std::vector presentedFieldsMask; + for (const auto& ruleField : M_RULE_FIELDS) { + if (!Rule::isWildcardRuleField(ruleField) && !Rule::isRegexRuleField(ruleField) + && !Rule::isIPRuleField(ruleField)) { + presentedFieldsMask.push_back(true); + } else { + presentedFieldsMask.push_back(false); + } + } + return presentedFieldsMask; +} + +} // namespace ListDetector diff --git a/modules/listDetector/src/rule.hpp b/modules/listDetector/src/rule.hpp new file mode 100644 index 00000000..cf728036 --- /dev/null +++ b/modules/listDetector/src/rule.hpp @@ -0,0 +1,137 @@ +/** + * @file + * @author Pavel Siska + * @author Daniel Pelanek + * @brief Defines data structures for a rule. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "ipAddressFieldMatcher.hpp" +#include "ipAddressPrefix.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ListDetector { + +/** + * @brief Stores statistics about a rule. + */ +struct RuleStats { + uint64_t matchedCount; /**< Number of times the rule has been matched. */ +}; + +/** + * @brief Represents possible values for a field in the rule. + */ +using RuleFieldValue = std::variant< + char, + uint8_t, + uint16_t, + uint32_t, + uint64_t, + int8_t, + int16_t, + int32_t, + int64_t, + std::string, + std::regex, + IpAddressPrefix>; + +/** + * @brief Represents a field in a rule. + */ +using RuleField = std::pair>; +/** + * @brief Represents hash value of the field in a rule. + */ +using HashValue = uint64_t; + +/** + * @brief Represents a single rule. + */ +class Rule { +public: + /** + * @brief Constructor for a Rule. + * @param ruleFields reference to a vector of rule fields. + */ + explicit Rule(std::vector ruleFields); + + /** + * @brief Checks if the given UnirecRecordView dynamic fields match dynamic fields of this rule. + * @param unirecRecordView The Unirec record which is tried to match. + * @return True if matched, false otherwise. + */ + bool dynamicFieldsMatch(const Nemea::UnirecRecordView& unirecRecordView); + + /** + * @brief Gets the statistics for this rule. + * @return A constant reference to the RuleStats structure. + */ + const RuleStats& getStats() const noexcept; + + /** + * @brief Checks if the given RuleField keeps static Unirec type. + * @param ruleField The RuleField to check. + * @return True if kept type is static, false otherwise. + */ + static bool isStaticRuleField(const RuleField& ruleField) noexcept; + + /** + * @brief Checks if the given RuleField represents wildcard value that matches everything. + * @param ruleField The RuleField to check. + * @return True if kept value is wildcard, false otherwise. + */ + static bool isWildcardRuleField(const RuleField& ruleField) noexcept; + + /** + * @brief Checks if the given RuleField represents regular expression. + * @param ruleField The RuleField to check. + * @return True if kept value is regex, false otherwise. + */ + static bool isRegexRuleField(const RuleField& ruleField) noexcept; + + /** + * @brief Checks if the given RuleField represents IP address. + * @param ruleField The RuleField to check. + * @return True if kept value is IP, false otherwise. + */ + static bool isIPRuleField(const RuleField& ruleField) noexcept; + + /** + * @brief Checks if the given RuleField represents string - normal string or regular expression. + * @param ruleField The RuleField to check. + * @return True if kept value is string, false otherwise. + */ + static bool isStringRuleField(const RuleField& ruleField) noexcept; + + /** + * @brief Getter for rule fields. + * @return Vector of rule fields. + */ + const std::vector& getRuleFields() const noexcept; + + /** + * @brief Calculates presented static fields mask. + * @return Bitset where presented static fields are set to true, regex, IP address or wildcard + * fields are set to false. + */ + std::vector getPresentedStaticFieldsMask() const noexcept; + +private: + const std::vector M_RULE_FIELDS; + + RuleStats m_stats {}; +}; + +} // namespace ListDetector From 14e18bd56fb2fa54b79e3017f843e162c39975c3 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sat, 24 Aug 2024 03:02:15 +0500 Subject: [PATCH 06/15] ListDetector - Introduce OctetNode structure --- modules/listDetector/src/octetNode.hpp | 45 ++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 modules/listDetector/src/octetNode.hpp diff --git a/modules/listDetector/src/octetNode.hpp b/modules/listDetector/src/octetNode.hpp new file mode 100644 index 00000000..8167369d --- /dev/null +++ b/modules/listDetector/src/octetNode.hpp @@ -0,0 +1,45 @@ +/** + * @file + * @author Damir Zainullin + * @brief Declares the OctetNode structure used to match IP addresses against prefixes. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include + +namespace ListDetector { + +/** + * @brief Keeps one octet of IP prefix as ip and mask octet. + */ +struct OctetNode { +public: + /** + * @brief Operator for comparing two OctetNodes. + * @param other The OctetNode to compare with. + * @return True if the two OctetNodes have same value and mask, but current node is terminating. + */ + bool operator==(const OctetNode& other) const noexcept + { + return value == other.value && mask == other.mask && !isLast; + } + + /** @brief Value signalizing that there is no next node. */ + inline static const uint16_t NO_INDEX = 0; + + std::byte value; ///< Value of IP prefix octet + std::byte mask; ///< Value of IP prefix mask octet corresponing to "value" octet + bool isLast; ///< True if this octet is last one for prefix (at maximum 4. for IPv4 and 16. for + ///< IPv6) + union { + uint16_t nextNode; ///< Index of next node + uint16_t rule; ///< Index of rule + } index; ///< Union that keeps index of next node for that IP prefix or index of rule which this + ///< prefix belongs to +}; + +} // namespace ListDetector From 1b67e682f4095b76795b1e0bfaf2fb2cd29362d9 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Tue, 13 Aug 2024 22:25:46 +0500 Subject: [PATCH 07/15] ListDetector - Introduce IpAddressFieldMatcher class --- .../src/ipAddressFieldMatcher.cpp | 141 ++++++++++++++++++ .../src/ipAddressFieldMatcher.hpp | 72 +++++++++ 2 files changed, 213 insertions(+) create mode 100644 modules/listDetector/src/ipAddressFieldMatcher.cpp create mode 100644 modules/listDetector/src/ipAddressFieldMatcher.hpp diff --git a/modules/listDetector/src/ipAddressFieldMatcher.cpp b/modules/listDetector/src/ipAddressFieldMatcher.cpp new file mode 100644 index 00000000..8cdd1146 --- /dev/null +++ b/modules/listDetector/src/ipAddressFieldMatcher.cpp @@ -0,0 +1,141 @@ +/** + * @file + * @author Damir Zainullin + * @brief Implementation of the IpAddressFieldMatcher class for keeping and matching IP prefixes + * against IP addresses + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "ipAddressFieldMatcher.hpp" +#include + +namespace ListDetector { + +static bool isNetworkMaskOctet(std::byte byte) noexcept +{ + const uint8_t networkMaskOctet = 0xFF; + return byte == std::byte(networkMaskOctet); +} + +void IpAddressFieldMatcher::addPrefix(const IpAddressPrefix& prefix) noexcept +{ + auto [ip, mask] = prefix.getIpAndMask(); + uint16_t previousOctetPos = std::numeric_limits::max(); + + for (auto octetIndex = 0U; octetIndex < ip.size(); octetIndex++) { + if (isNetworkMaskOctet(mask[octetIndex]) && octetIndex != ip.size() - 1) { + previousOctetPos = insertNode( + OctetNode {ip[octetIndex], mask[octetIndex], false, {OctetNode::NO_INDEX}}, + NodePos {(uint8_t) octetIndex, previousOctetPos}); + } else { + insertNode( + OctetNode {ip[octetIndex], mask[octetIndex], true, {m_lastInsertIndex++}}, + NodePos {(uint8_t) octetIndex, previousOctetPos}); + return; + } + } +} + +uint16_t IpAddressFieldMatcher::insertNode(const OctetNode& node, NodePos pos) noexcept +{ + const auto& [octetIndex, previousOctetPos] = pos; + + const uint16_t startIndex + = octetIndex == 0 ? 0 : m_octets[octetIndex - 1UL][previousOctetPos].index.nextNode; + const uint16_t endIndex = getNotLastNodeNextNode(NodePos {octetIndex, previousOctetPos}); + + for (auto index = startIndex; index < endIndex; index++) { + if (m_octets[octetIndex][index] == node) { + return index; + } + } + + m_octets[octetIndex].insert(m_octets[octetIndex].begin() + startIndex, node); + + for (auto i = previousOctetPos + 1U; octetIndex != 0 && i < m_octets[octetIndex - 1UL].size(); + i++) { + if (!m_octets[octetIndex - 1UL][i].isLast) { + m_octets[octetIndex - 1UL][i].index.nextNode++; + } + } + + return startIndex; +} + +std::vector IpAddressFieldMatcher::getMatchingIpRulesMask( + const Nemea::IpAddress& address, + const std::vector& previouslyMatchedRulesMask) const +{ + std::vector matchingRulesMask(m_lastInsertIndex); + checkOctet(address, 0, {0, m_octets[0].size()}, matchingRulesMask, previouslyMatchedRulesMask); + return matchingRulesMask; +} + +uint16_t IpAddressFieldMatcher::getNotLastNodeNextNode(NodePos pos) const noexcept +{ + const auto& [octetIndex, startIndex] = pos; + + if (octetIndex == 0) { + return (uint16_t)m_octets[octetIndex].size(); + } + auto index = startIndex + 1U; + for (; index < m_octets[octetIndex - 1UL].size() && m_octets[octetIndex - 1UL][index].isLast; + index++) {} + if (index == m_octets[octetIndex - 1UL].size()) { + return (uint16_t)m_octets[octetIndex].size(); + } + return m_octets[octetIndex - 1UL][index].index.nextNode; +} + +void IpAddressFieldMatcher::checkOctet( + const Nemea::IpAddress& address, + uint8_t octetIndex, + const std::pair& searchRange, + std::vector& matchingBitset, + const std::vector& previouslyMatchedBitset) const noexcept +{ + auto [startIndex, endIndex] = searchRange; + + for (auto i = startIndex; i < endIndex; i++) { + const bool currentOctetIsLast = m_octets[octetIndex][i].isLast; + const bool octetMatches = ipMatchesNetworkOctet(address, NodePos {octetIndex, i}); + + if (octetMatches && currentOctetIsLast + && previouslyMatchedBitset[m_octets[octetIndex][i].index.rule]) { + matchingBitset[m_octets[octetIndex][i].index.rule] = true; + } else if (octetMatches) { + auto nextOctetSearchEndIndex + = getNotLastNodeNextNode(NodePos {(uint8_t) (octetIndex + 1), i}); + checkOctet( + address, + static_cast(octetIndex + 1UL), + {m_octets[octetIndex][i].index.nextNode, nextOctetSearchEndIndex}, + matchingBitset, + previouslyMatchedBitset); + } + } +} + +bool IpAddressFieldMatcher::ipMatchesNetworkOctet(const Nemea::IpAddress& address, NodePos pos) + const noexcept +{ + const auto& [octetIndex, nodeIndex] = pos; + + if (address.isIpv4()) { + return m_octets[octetIndex][nodeIndex].value + == (std::byte)( + ip_get_v4_as_bytes(&address.ip)[octetIndex] + & (uint8_t) m_octets[octetIndex][nodeIndex].mask); + } + return m_octets[octetIndex][nodeIndex].value + == (std::byte)( + address.ip.bytes[octetIndex] & (uint8_t) m_octets[octetIndex][nodeIndex].mask); +} + +void IpAddressFieldMatcher::addEmptyPrefix() noexcept +{ + addPrefix(IpAddressPrefix(Nemea::IpAddress {}, 0)); +} + +} // namespace ListDetector diff --git a/modules/listDetector/src/ipAddressFieldMatcher.hpp b/modules/listDetector/src/ipAddressFieldMatcher.hpp new file mode 100644 index 00000000..d21911f5 --- /dev/null +++ b/modules/listDetector/src/ipAddressFieldMatcher.hpp @@ -0,0 +1,72 @@ +/** + * @file + * @author Damir Zainullin + * @brief Declaration of the IpAddressFieldMatcher class for keeping and matching IP prefixes + * against IP addresses + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "ipAddressPrefix.hpp" +#include "octetNode.hpp" + +#include +#include +#include +#include + +namespace ListDetector { + +/** + * @brief Keeps IP address prefixes and match IP addresses against them. + */ +class IpAddressFieldMatcher { +public: + /** + * @brief Adds given IP prefix to the address matcher. + * @param prefix The IP prefix to add. + */ + void addPrefix(const IpAddressPrefix& prefix) noexcept; + + /** + * @brief Adds empty prefix to the address matcher to match all adresses. + */ + void addEmptyPrefix() noexcept; + + /** + * @brief Finds IP prefixes mathing given IP address ignoring IP addresses from rules that can + * not match. + * @param address The IP adress to match prefixes against. + * @param previouslyMatchedRulesMask Bitset of previously matched rules. + * @return Bitset where matching IP prefix indexes are set to true. + */ + std::vector getMatchingIpRulesMask( + const Nemea::IpAddress& address, + const std::vector& previouslyMatchedRulesMask) const; + +private: + struct NodePos { + uint8_t octetIndex; ///< Index of the octet in the IP address. Allowed values are in range + ///< [0,15]. + uint16_t nodeIndex; ///< Index of the node on the level octetIndex in the tree. + }; + + uint16_t getNotLastNodeNextNode(NodePos pos) const noexcept; + uint16_t insertNode(const OctetNode& node, NodePos pos) noexcept; + void checkOctet( + const Nemea::IpAddress& address, + uint8_t octetIndex, + const std::pair& searchRange, + std::vector& matchingBitset, + const std::vector& previouslyMatchedBitset) const noexcept; + bool ipMatchesNetworkOctet(const Nemea::IpAddress& address, NodePos pos) const noexcept; + + inline static const int OCTET_MAX_COUNT = 16; + std::array, OCTET_MAX_COUNT> m_octets; + + uint16_t m_lastInsertIndex = 0; +}; + +} // namespace ListDetector From b3d8bf01fe4c1b3c46751bda38a21d4bf03c8bcf Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 10 Nov 2024 05:00:07 +0100 Subject: [PATCH 08/15] List Detector - Introduce FieldsMatcher class --- modules/listDetector/src/fieldsMatcher.cpp | 202 +++++++++++++++++++++ modules/listDetector/src/fieldsMatcher.hpp | 61 +++++++ 2 files changed, 263 insertions(+) create mode 100644 modules/listDetector/src/fieldsMatcher.cpp create mode 100644 modules/listDetector/src/fieldsMatcher.hpp diff --git a/modules/listDetector/src/fieldsMatcher.cpp b/modules/listDetector/src/fieldsMatcher.cpp new file mode 100644 index 00000000..5f60f58c --- /dev/null +++ b/modules/listDetector/src/fieldsMatcher.cpp @@ -0,0 +1,202 @@ +/** + * @file + * @author Damir Zainullin + * @brief Implementation of the FieldsMatcher class for creating hash values of static fields of + * Unirec records or rules and matching their dynamic fields + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "fieldsMatcher.hpp" + +#include +#include +#include +#include + +namespace ListDetector { + +static std::pair +getStaticFieldAsCharPtr(const Nemea::UnirecRecordView& unirecRecordView, ur_field_id_t fieldId) +{ + switch (ur_get_type(fieldId)) { + case UR_TYPE_CHAR: + return {unirecRecordView.getFieldAsType(fieldId), sizeof(char)}; + case UR_TYPE_UINT8: + return { + reinterpret_cast(unirecRecordView.getFieldAsType(fieldId)), + sizeof(uint8_t)}; + case UR_TYPE_INT8: + return { + reinterpret_cast(unirecRecordView.getFieldAsType(fieldId)), + sizeof(int8_t)}; + case UR_TYPE_UINT16: + return { + reinterpret_cast( + unirecRecordView.getFieldAsType(fieldId)), + sizeof(uint16_t)}; + case UR_TYPE_INT16: + return { + reinterpret_cast(unirecRecordView.getFieldAsType(fieldId)), + sizeof(int16_t)}; + case UR_TYPE_UINT32: + return { + reinterpret_cast( + unirecRecordView.getFieldAsType(fieldId)), + sizeof(uint32_t)}; + case UR_TYPE_INT32: + return { + reinterpret_cast(unirecRecordView.getFieldAsType(fieldId)), + sizeof(int32_t)}; + case UR_TYPE_UINT64: + return { + reinterpret_cast( + unirecRecordView.getFieldAsType(fieldId)), + sizeof(uint64_t)}; + case UR_TYPE_INT64: + return { + reinterpret_cast(unirecRecordView.getFieldAsType(fieldId)), + sizeof(int64_t)}; + case UR_TYPE_STRING: { + auto str = unirecRecordView.getFieldAsType(fieldId); + return {str.data(), str.length()}; + } + default: + throw std::invalid_argument( + std::string("Given field id ") + std::to_string(fieldId) + " is not a static field"); + } + return {nullptr, 0}; +} + +static void writeFieldToBuffer( + std::vector& buffer, + size_t& writePos, + const Nemea::UnirecRecordView& unirecRecordView, + ur_field_id_t fieldId) +{ + auto [ptr, length] = getStaticFieldAsCharPtr(unirecRecordView, fieldId); + if (writePos + length >= buffer.capacity()) { + buffer.resize(writePos + (4 * length)); + } + std::memcpy(buffer.data() + writePos, ptr, length); + writePos += length; +} + +uint64_t FieldsMatcher::calculateStaticHash( + const Nemea::UnirecRecordView& unirecRecordView, + const std::vector& presentedStaticFieldsMask) +{ + size_t writePos = 0; + for (auto fieldIndex = 0U; fieldIndex < m_fieldIds.size(); fieldIndex++) { + if (!presentedStaticFieldsMask[fieldIndex]) { + continue; + } + writeFieldToBuffer(m_buffer, writePos, unirecRecordView, m_fieldIds[fieldIndex]); + } + return XXH64(m_buffer.data(), writePos, 0); +} + +FieldsMatcher::FieldsMatcher(std::vector& rules) + : m_rules(rules) +{ + if (!rules.empty()) { + std::transform( + rules.begin()->getRuleFields().begin(), + rules.begin()->getRuleFields().end(), + std::back_inserter(m_fieldIds), + [](const RuleField& ruleField) { return ruleField.first; }); + } + + for (const auto& rule : rules) { + resizeHashBuffer(rule); + m_rulesStaticHashIndexes.insert(std::make_pair(calculateStaticHash(rule), m_ruleIndex++)); + m_presentedStaticFieldsMasks.emplace(rule.getPresentedStaticFieldsMask()); + } +} + +void FieldsMatcher::resizeHashBuffer(const Rule& rule) +{ + const size_t totalBufferLength = std::accumulate( + rule.getRuleFields().begin(), + rule.getRuleFields().end(), + 0UL, + [](uint32_t length, const auto& ruleField) -> size_t { + // Wildcard and regex rule fields are not included in the hash value + if (Rule::isWildcardRuleField(ruleField) || Rule::isRegexRuleField(ruleField) + || Rule::isIPRuleField(ruleField)) { + return length; + } + if (Rule::isStaticRuleField(ruleField)) { + return length + sizeof(size_t); + } + if (Rule::isStringRuleField(ruleField) && !Rule::isRegexRuleField(ruleField)) { + return length + std::get(ruleField.second.value()).length(); + } + throw std::runtime_error("Unexpected rule field type"); + }); + m_buffer.resize(totalBufferLength); +} + +bool FieldsMatcher::anyOfRulesMatch( + const Nemea::UnirecRecordView& unirecRecordView, + const std::vector& previouslyMatchedRulesMask) +{ + bool match = false; + + for (const auto& presentedStaticFieldsMask : m_presentedStaticFieldsMasks) { + const size_t hashValue = calculateStaticHash(unirecRecordView, presentedStaticFieldsMask); + + for (auto [it, rangeEnd] = m_rulesStaticHashIndexes.equal_range(hashValue); + !match && it != rangeEnd; + it++) { + match = previouslyMatchedRulesMask[it->second] + && m_rules[it->second].dynamicFieldsMatch(unirecRecordView); + } + if (match) { + break; + } + } + return match; +} + +struct StaticFieldsHashVisitor { + StaticFieldsHashVisitor(std::byte* buffer, size_t& writePos) + : m_buffer(buffer) + , m_writePos(writePos) + { + } + + template + void operator()(const T& value) + { + if constexpr (std::is_trivially_copyable_v) { + std::memcpy(m_buffer + m_writePos, &value, sizeof(value)); + m_writePos += sizeof(value); + } else if constexpr (std::is_same_v) { + std::memcpy(m_buffer + m_writePos, value.c_str(), value.length()); + m_writePos += value.length(); + } + } + +private: + std::byte* m_buffer; + size_t& m_writePos; +}; + +uint64_t FieldsMatcher::calculateStaticHash(const Rule& rule) +{ + size_t writePos = 0; + for (const auto& ruleField : rule.getRuleFields()) { + if (Rule::isWildcardRuleField(ruleField) || Rule::isRegexRuleField(ruleField) + || Rule::isIPRuleField(ruleField)) { + continue; + } + if (const std::optional& ruleFieldOpt = ruleField.second; + ruleFieldOpt.has_value()) { + std::visit(StaticFieldsHashVisitor {m_buffer.data(), writePos}, *ruleFieldOpt); + } + } + return XXH64(m_buffer.data(), writePos, 0); +} + +} // namespace ListDetector diff --git a/modules/listDetector/src/fieldsMatcher.hpp b/modules/listDetector/src/fieldsMatcher.hpp new file mode 100644 index 00000000..c8b2117e --- /dev/null +++ b/modules/listDetector/src/fieldsMatcher.hpp @@ -0,0 +1,61 @@ +/** + * @file + * @author Damir Zainullin + * @brief Declaration of the FieldsMatcher class for creating hash values of static fields of + * Unirec records or rules and matching their dynamic fields + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "rule.hpp" + +#include +#include +#include +#include +#include +#include + +namespace ListDetector { + +/** + * @brief Matches fields of Unirec records against UnirecordViews by hash value. + */ +class FieldsMatcher { +public: + /** + * @brief Constructor for a StaticFieldsHasher. + * @param rules reference to a vector of rule fields id. + */ + explicit FieldsMatcher(std::vector& rules); + + /** + * @brief Checks if some rule matches given Unirec view. + * @param unirecRecordView The Unirec record view to find matching rules. + * @param previouslyMatchedRulesMask Bitset of previously matched rules. + * @return True if some rule matched, false otherwise. + */ + bool anyOfRulesMatch( + const Nemea::UnirecRecordView& unirecRecordView, + const std::vector& previouslyMatchedRulesMask); + +private: + void resizeHashBuffer(const Rule& rule); + uint64_t calculateStaticHash( + const Nemea::UnirecRecordView& unirecRecordView, + const std::vector& presentedStaticFieldsMask); + uint64_t calculateStaticHash(const Rule& rule); + + std::vector& m_rules; + std::unordered_multimap m_rulesStaticHashIndexes; + std::vector m_fieldIds; + std::unordered_set> m_presentedStaticFieldsMasks; + + std::vector m_buffer; + + uint16_t m_ruleIndex = 0; +}; + +} // namespace ListDetector From cf78cb1d5d30812565b98b9385ffb595d34c5557 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Fri, 20 Sep 2024 16:05:33 +0200 Subject: [PATCH 09/15] ListDetector - Introduce RulesMatcher class --- modules/listDetector/src/rulesMatcher.cpp | 55 +++++++++++++++++++++++ modules/listDetector/src/rulesMatcher.hpp | 50 +++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 modules/listDetector/src/rulesMatcher.cpp create mode 100644 modules/listDetector/src/rulesMatcher.hpp diff --git a/modules/listDetector/src/rulesMatcher.cpp b/modules/listDetector/src/rulesMatcher.cpp new file mode 100644 index 00000000..8c7216be --- /dev/null +++ b/modules/listDetector/src/rulesMatcher.cpp @@ -0,0 +1,55 @@ +/** + * @file + * @author Damir Zainullin + * @brief Implementation the RulesMatcher class for matching rules against Unirec views. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "rulesMatcher.hpp" +#include "ruleBuilder.hpp" + +namespace ListDetector { + +RulesMatcher::RulesMatcher(const ConfigParser* configParser) noexcept +{ + const std::string unirecTemplateDescription = configParser->getUnirecTemplateDescription(); + + RuleBuilder ruleBuilder(unirecTemplateDescription); + + for (const auto& ruleDescription : configParser->getRulesDescription()) { + auto rule = ruleBuilder.build(ruleDescription); + m_rules.emplace_back(rule); + } + + m_ipAddressFieldMatchers = ruleBuilder.getIpAddressFieldMatchers(); + m_fieldsMatcher = std::make_unique(m_rules); +} + +std::vector +RulesMatcher::getMatchingIpRulesMask(const Nemea::UnirecRecordView& unirecRecordView) +{ + std::vector previouslyMatchedRulesMask(m_rules.size(), true); + + for (auto& [fieldId, ipAddressMatcher] : *m_ipAddressFieldMatchers) { + const auto& ipAddress = unirecRecordView.getFieldAsType(fieldId); + auto matchingRulesMask + = ipAddressMatcher.getMatchingIpRulesMask(ipAddress, previouslyMatchedRulesMask); + previouslyMatchedRulesMask = std::move(matchingRulesMask); + } + + return previouslyMatchedRulesMask; +} + +bool RulesMatcher::anyOfRuleMatches(const Nemea::UnirecRecordView& unirecRecordView) +{ + auto matchingIpRulesMask = getMatchingIpRulesMask(unirecRecordView); + return m_fieldsMatcher->anyOfRulesMatch(unirecRecordView, matchingIpRulesMask); +} + +std::vector& RulesMatcher::getRules() noexcept +{ + return m_rules; +} + +} // namespace ListDetector diff --git a/modules/listDetector/src/rulesMatcher.hpp b/modules/listDetector/src/rulesMatcher.hpp new file mode 100644 index 00000000..a0a796d5 --- /dev/null +++ b/modules/listDetector/src/rulesMatcher.hpp @@ -0,0 +1,50 @@ +/** + * @file + * @author Damir Zainullin + * @brief Declares the RulesMatcher class for matching rules against Unirec views. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "configParser.hpp" +#include "fieldsMatcher.hpp" + +namespace ListDetector { + +/** + * @brief RulesMatcher class to match unirec records against prefix IP trees and rule hashes. + */ +class RulesMatcher { +public: + /** + * @brief Constructor for a RulesMatcher. + * @param configParser pointer to config parser. + */ + explicit RulesMatcher(const ConfigParser* configParser) noexcept; + + /** + * @brief Checks if some rule matches given Unirec view. + * @param unirecRecordView The Unirec view to match. + * @return True if some rule matched, false otherwise. + */ + bool anyOfRuleMatches(const Nemea::UnirecRecordView& unirecRecordView); + + /** + * @brief Getter for kept rules. + * @return Vector of rules. + */ + std::vector& getRules() noexcept; + +private: + std::vector getMatchingIpRulesMask(const Nemea::UnirecRecordView& unirecRecordView); + + std::vector m_rules; + + std::shared_ptr> + m_ipAddressFieldMatchers; + std::unique_ptr m_fieldsMatcher; +}; + +} // namespace ListDetector From 2f0ed45d253b9fdc81e71f1e8a3c3e8725bdee9d Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 11 Aug 2024 15:01:52 +0500 Subject: [PATCH 10/15] ListDetector - Introduce ListDetector class --- modules/listDetector/src/listDetector.cpp | 85 +++++++++++++++++++++++ modules/listDetector/src/listDetector.hpp | 70 +++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 modules/listDetector/src/listDetector.cpp create mode 100644 modules/listDetector/src/listDetector.hpp diff --git a/modules/listDetector/src/listDetector.cpp b/modules/listDetector/src/listDetector.cpp new file mode 100644 index 00000000..dfbe252e --- /dev/null +++ b/modules/listDetector/src/listDetector.cpp @@ -0,0 +1,85 @@ +/** + * @file + * @author Pavel Siska + * @author Daniel Pelanek + * @brief Implementation of the ListDetector class. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "listDetector.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ListDetector { + +static telemetry::Content createRuleTelemetryContent(const Rule& rule) +{ + const RuleStats& ruleStats = rule.getStats(); + telemetry::Dict dict; + dict["matchedCount"] = telemetry::Scalar(ruleStats.matchedCount); + return dict; +} + +ListDetectorMode ListDetector::convertStringToListDetectorMode(const std::string& str) +{ + if (str != "bl" && str != "wl" && str != "blacklist" && str != "whitelist") { + throw std::runtime_error( + "Unknown list mode. Only allowed values are blacklist and whitelist"); + } + + if (str == "bl" || str == "blacklist") { + return ListDetectorMode::BLACKLIST; + } + return ListDetectorMode::WHITELIST; +} + +ListDetector::ListDetector(const ConfigParser* configParser, ListDetectorMode mode) + : m_mode(mode) + , m_rulesMatcher(configParser) +{ +} + +bool ListDetector::matches(const Nemea::UnirecRecordView& unirecRecordView) +{ + const bool match = m_rulesMatcher.anyOfRuleMatches(unirecRecordView); + + if (m_mode == ListDetectorMode::WHITELIST) { + return match; + } + return !match; +} + +void ListDetector::setTelemetryDirectory(const std::shared_ptr& directory) +{ + m_holder.add(directory); + + auto rulesDirectory = directory->addDir("rules"); + const std::vector& rules = m_rulesMatcher.getRules(); + + for (size_t ruleIndex = 0; ruleIndex < rules.size(); ruleIndex++) { + const Rule& rule = rules.at(ruleIndex); + const telemetry::FileOps fileOps + = {[&rule]() { return createRuleTelemetryContent(rule); }, nullptr}; + auto ruleFile = rulesDirectory->addFile(std::to_string(ruleIndex), fileOps); + m_holder.add(ruleFile); + } + + const telemetry::AggOperation aggFileOps = { + telemetry::AggMethodType::SUM, + "matchedCount", + "totalMatchedCount", + }; + + auto aggFile = directory->addAggFile("aggStats", "rules/.*", {aggFileOps}); + m_holder.add(aggFile); +} + +} // namespace ListDetector diff --git a/modules/listDetector/src/listDetector.hpp b/modules/listDetector/src/listDetector.hpp new file mode 100644 index 00000000..69d7d078 --- /dev/null +++ b/modules/listDetector/src/listDetector.hpp @@ -0,0 +1,70 @@ +/** + * @file + * @author Pavel Siska + * @author Daniel Pelanek + * @brief Declaration of the ListDetector class. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "configParser.hpp" +#include "rulesMatcher.hpp" + +#include +#include +#include +#include + +namespace ListDetector { + +/** + * @brief Possible modes of the ListDetector. + */ +enum class ListDetectorMode : uint8_t { + BLACKLIST, ///< Only records that match some rule in rule list are forwarded. + WHITELIST ///< Only records that do not match any rule in rule list are forwarded. +}; + +/** + * @brief Represents a ListDetector for Nemea++ records. + */ +class ListDetector { +public: + /** + * @brief Constructor for ListDetector. + * @param configParser Pointer to the ConfigParser providing rules. + * @param mode Mode to use. + */ + explicit ListDetector(const ConfigParser* configParser, ListDetectorMode mode); + + /** + * @brief Checks if the given UnirecRecordView matches some rule from ListDetector. + * @param unirecRecordView The Unirec record to check against the ListDetector. + * @return True if matches, false otherwise. + */ + bool matches(const Nemea::UnirecRecordView& unirecRecordView); + + /** + * @brief Sets the telemetry directory for the ListDetector. + * @param directory directory for ListDetector telemetry. + */ + void setTelemetryDirectory(const std::shared_ptr& directory); + + /** + * @brief Converts provided string to the list detector mode. + * @param str String to convert. + * @return List derector mode specified by string. + */ + static ListDetectorMode convertStringToListDetectorMode(const std::string& str); + +private: + telemetry::Holder m_holder; + + ListDetectorMode m_mode; + + RulesMatcher m_rulesMatcher; +}; + +} // namespace ListDetector From 025d07f23e3fa7e5bce1a734a7fa73b8f1b6c7fd Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 11 Aug 2024 15:03:12 +0500 Subject: [PATCH 11/15] ListDetector - Adjust main.cpp --- modules/listDetector/src/main.cpp | 201 ++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 modules/listDetector/src/main.cpp diff --git a/modules/listDetector/src/main.cpp b/modules/listDetector/src/main.cpp new file mode 100644 index 00000000..8954e1cd --- /dev/null +++ b/modules/listDetector/src/main.cpp @@ -0,0 +1,201 @@ +/** + * @file + * @author Pavel Siska + * @brief ListDetector Module: Process and filter Unirec records based on rules. + * + * This file contains the main function and supporting functions for the Unirec ListDetector Module. + * The module processes Unirec records through a bidirectional interface, checking against a + * list of rules, and forwarding non-whitelisted records or blacklisted records. It utilizes the + * Unirec++ library for record handling, argparse for command-line argument parsing, and various + * custom classes for configuration parsing, logging, and whitelist or blacklist rule checking. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "csvConfigParser.hpp" +#include "listDetector.hpp" +#include "logger/logger.hpp" +#include "unirec/unirec-telemetry.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Nemea; + +std::atomic g_stopFlag(false); + +void signalHandler(int signum) +{ + Nm::loggerGet("signalHandler")->info("Interrupt signal {} received", signum); + g_stopFlag.store(true); +} + +/** + * @brief Handle a format change exception by adjusting the template. + * + * This function is called when a `FormatChangeException` is caught in the main loop. + * It adjusts the template in the bidirectional interface to handle the format change. + * + * @param biInterface Bidirectional interface for Unirec communication. + */ +static void handleFormatChange(UnirecBidirectionalInterface& biInterface) +{ + biInterface.changeTemplate(); +} + +/** + * @brief Process the next Unirec record and forward if blacklisted or not whitelisted. + * + * This function receives the next Unirec record through the bidirectional interface. + * If the record matches the list, it is forwarded using the same interface. + * + * @param biInterface Bidirectional interface for Unirec communication. + * @param listDetector ListDetector instance for checking Unirec records. + */ +static void processNextRecord( + UnirecBidirectionalInterface& biInterface, + ListDetector::ListDetector& listDetector) +{ + std::optional unirecRecord = biInterface.receive(); + if (!unirecRecord) { + return; + } + + if (!listDetector.matches(*unirecRecord)) { + biInterface.send(*unirecRecord); + } +} + +/** + * @brief Process Unirec records based on the rules of listDetector. + * + * The `processUnirecRecords` function continuously receives Unirec records through the provided + * bidirectional interface (`biInterface`). Each received record is checked against the + * specified rule list (blacklist or whitelist). If the record matches the list, it is forwarded + * using the bidirectional interface. The loop runs indefinitely until an end-of-file condition is + * encountered. + * + * @param biInterface Bidirectional interface for Unirec communication. + * @param listDetector ListDetector instance for checking Unirec records. + */ +static void processUnirecRecords( + UnirecBidirectionalInterface& biInterface, + ListDetector::ListDetector& listDetector) +{ + while (!g_stopFlag.load()) { + try { + processNextRecord(biInterface, listDetector); + } catch (FormatChangeException& ex) { + handleFormatChange(biInterface); + } catch (const EoFException& ex) { + break; + } catch (const std::exception& ex) { + throw; + } + } +} + +int main(int argc, char** argv) +{ + argparse::ArgumentParser program("listdetector"); + + Nm::loggerInit(); + auto logger = Nm::loggerGet("main"); + + signal(SIGINT, signalHandler); + + try { + program.add_argument("-r", "--rules") + .required() + .help("specify the CSV rule file.") + .metavar("csv_file"); + + program.add_argument("-lm", "--listmode") + .help("specify the list detector mode. Default is whitelist") + .default_value(std::string("whitelist")); + + program.add_argument("-m", "--appfs-mountpoint") + .required() + .help("path where the appFs directory will be mounted") + .default_value(std::string("")); + } catch (std::exception& ex) { + logger->error(ex.what()); + return EXIT_FAILURE; + } + + Unirec unirec({1, 1, "ListDetector", "Unirec list detector module"}); + + try { + unirec.init(argc, argv); + } catch (const HelpException& ex) { + std::cerr << program; + return EXIT_SUCCESS; + } catch (const std::exception& ex) { + logger->error(ex.what()); + return EXIT_FAILURE; + } + + try { + program.parse_args(argc, argv); + } catch (const std::exception& ex) { + logger->error(ex.what()); + return EXIT_FAILURE; + } + + std::shared_ptr telemetryRootDirectory; + telemetryRootDirectory = telemetry::Directory::create(); + + std::unique_ptr appFs; + + try { + auto mountPoint = program.get("--appfs-mountpoint"); + if (!mountPoint.empty()) { + const bool tryToUnmountOnStart = true; + const bool createMountPoint = true; + appFs = std::make_unique( + telemetryRootDirectory, + mountPoint, + tryToUnmountOnStart, + createMountPoint); + appFs->start(); + } + } catch (std::exception& ex) { + logger->error(ex.what()); + return EXIT_FAILURE; + } + + try { + std::unique_ptr configParser + = std::make_unique(program.get("--rules")); + const std::string requiredUnirecTemplate = configParser->getUnirecTemplateDescription(); + + UnirecBidirectionalInterface biInterface = unirec.buildBidirectionalInterface(); + biInterface.setRequieredFormat(requiredUnirecTemplate); + + auto telemetryInputDirectory = telemetryRootDirectory->addDir("input"); + const telemetry::FileOps inputFileOps + = {[&biInterface]() { return Nm::getInterfaceTelemetry(biInterface); }, nullptr}; + const auto inputFile = telemetryInputDirectory->addFile("stats", inputFileOps); + + auto mode = ListDetector::ListDetector::convertStringToListDetectorMode( + program.get("--listmode")); + + ListDetector::ListDetector listDetector(configParser.get(), mode); + auto listDetectorTelemetryDirectory = telemetryRootDirectory->addDir("listdetector"); + listDetector.setTelemetryDirectory(listDetectorTelemetryDirectory); + + processUnirecRecords(biInterface, listDetector); + + } catch (std::exception& ex) { + logger->error(ex.what()); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} From 401bf07ec4f228288bcc4a685b85f6432158edc2 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 10 Nov 2024 04:01:10 +0100 Subject: [PATCH 12/15] ListDetector - Adjust CMakeLists --- modules/CMakeLists.txt | 1 + modules/listDetector/CMakeLists.txt | 1 + modules/listDetector/src/CMakeLists.txt | 26 +++++++++++++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 modules/listDetector/CMakeLists.txt create mode 100644 modules/listDetector/src/CMakeLists.txt diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt index ce0e2c7c..df464f25 100644 --- a/modules/CMakeLists.txt +++ b/modules/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(listDetector) add_subdirectory(sampler) add_subdirectory(telemetry) add_subdirectory(deduplicator) diff --git a/modules/listDetector/CMakeLists.txt b/modules/listDetector/CMakeLists.txt new file mode 100644 index 00000000..febd4f0a --- /dev/null +++ b/modules/listDetector/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(src) diff --git a/modules/listDetector/src/CMakeLists.txt b/modules/listDetector/src/CMakeLists.txt new file mode 100644 index 00000000..a19007d1 --- /dev/null +++ b/modules/listDetector/src/CMakeLists.txt @@ -0,0 +1,26 @@ +add_executable(listDetector + main.cpp + configParser.cpp + csvConfigParser.cpp + ipAddressPrefix.cpp + rule.cpp + ruleBuilder.cpp + listDetector.cpp + ipAddressFieldMatcher.cpp + fieldsMatcher.cpp + rulesMatcher.cpp +) + +target_link_libraries(listDetector PRIVATE + telemetry::telemetry + telemetry::appFs + common + rapidcsv + unirec::unirec++ + unirec::unirec + trap::trap + argparse + xxhash +) + +install(TARGETS listDetector DESTINATION ${INSTALL_DIR_BIN}) From c48bde396ee1a9c3471df3ca912a6aad0ba3c90a Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 10 Nov 2024 04:01:48 +0100 Subject: [PATCH 13/15] ListDetector - Adjust RMP build --- README.md | 1 + modules/listDetector/README.md | 84 ++++++++++++++++++++++++++++++++ pkg/rpm/nemea-modules-ng.spec.in | 1 + 3 files changed, 86 insertions(+) create mode 100644 modules/listDetector/README.md diff --git a/README.md b/README.md index 5bae4244..80841610 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ This repository contains basic modules of the [NEMEA system](https://github.com/CESNET/Nemea). The modules and their functionality/purposes are: +* [ListDetector](modules/listdetector/): forwards records that match rules list. * [Sampler](modules/sampler/): sample records at the given rate. * [Telemetry](modules/telemetry/): provides unirec telemetry of the input interface. * [Deduplicator](modules/deduplicator/): omit duplicate records. diff --git a/modules/listDetector/README.md b/modules/listDetector/README.md new file mode 100644 index 00000000..901b1715 --- /dev/null +++ b/modules/listDetector/README.md @@ -0,0 +1,84 @@ +# ListDetector module - README + +## Description +The module analyzes Unirec records by comparing them against a set of predefined rules in a rule list. +Rule list can be blacklist or whitelist. It identifies and forwards records that do not match to the whitelist rules or match blacklist rules. + +## Interfaces +- Input: 1 +- Output: 1 + +## Parameters +### Common TRAP parameters +- `-h [trap,1]` Print help message for this module / for libtrap specific parameters. +- `-i IFC_SPEC` Specification of interface types and their parameters. +- `-v` Be verbose. +- `-vv` Be more verbose. +- `-vvv` Be even more verbose. + +### Module specific parameters +- `-r, --rules ` ListDetector module rules in CSV format +- `-lm, --listmode ` ListDetector mode - whitelist or blacklist +- `-m, --appfs-mountpoint ` Path where the appFs directory will be mounted + +## CSV rules format +The first row of CSV specifies the unirec types and names of fields that will be +used for whitelisting or blacklisting. + +The supported unirec types are: `uint8`, `int8`, `uint16`, `int16`, `uint32`, `int32`, +`uint64`, `int64`, `char`, `ipaddr` and `string`. + +- Empty values match everyting. + +- Numeric types match the exact value. + +- IP address (`ipaddr`) can be either ipv4 or ipv6 address. +The ip address can optionally have a prefix. +If there is no prefix, the address must match exactly. + - Examples: `127.0.0.1`, `127.0.0.0/24` + +- String match a regex pattern. Regex patterns support extended grep syntax. + - Examples: `R"(^www.google.com$)"`, `R"(.*google\.com$)"` + +### Example CSV file + +``` +ipaddr SRC_IP,uint16 DST_PORT,uint16 SRC_PORT +10.0.0.1,443,53530 +10.0.0.2,443,53531 +``` + +``` +ipaddr SCR_IP,string QUIC_SNI +10.0.0.1/24,R"(.*google\.com$)" +``` + +## Usage Examples +``` +# Data from the input unix socket interface "trap_in" is processed, and entries that +do not match the defined rules in the "csvWhitelist.csv" file are forwarded to the +output interface "trap_out." + +$ listDetector -i u:trap_in,u:trap_out -r csvWhitelist.csv +``` +``` +# Data from the input unix socket interface "trap_in" is processed, and entries that +match the defined rules in the "csvblacklist.csv" file are forwarded to the +output interface "trap_out." + +$ listDetector -i u:trap_in,u:trap_out -lm bl -r csvBlacklist.csv +``` + +## Telemetry data format +``` +├─ input/ +│ └─ stats +└─ listDetector/ + ├─ aggStats + └─ rules/ + ├─ 0 + ├─ 1 + └ ... +``` + +Each rule has its own file named according to the order of the rules in the configuration file. diff --git a/pkg/rpm/nemea-modules-ng.spec.in b/pkg/rpm/nemea-modules-ng.spec.in index bcdd383b..bb06d98e 100644 --- a/pkg/rpm/nemea-modules-ng.spec.in +++ b/pkg/rpm/nemea-modules-ng.spec.in @@ -36,6 +36,7 @@ that make up the main components of the test environment. %files %license LICENSE +%{_bindir}/nemea/listDetector %{_bindir}/nemea/sampler %{_bindir}/nemea/telemetry_stats %{_bindir}/nemea/deduplicator From aec55c5d52b9ddf4ac586a74f1571f5f57035ce6 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Wed, 5 Feb 2025 00:15:04 +0100 Subject: [PATCH 14/15] List Detector - Add tests --- CMakeLists.txt | 5 ++ cmake/tests.cmake | 27 ++++++++ modules/listDetector/tests/test.sh | 62 +++++++++++++++++++ .../tests/testsData/inputs/input23.csv | 20 ++++++ .../tests/testsData/inputs/input24.csv | 18 ++++++ .../tests/testsData/inputs/input25.csv | 3 + .../tests/testsData/inputs/input26.csv | 47 ++++++++++++++ .../tests/testsData/inputs/input27.csv | 19 ++++++ .../tests/testsData/inputs/input28.csv | 41 ++++++++++++ .../tests/testsData/inputs/input29.csv | 8 +++ .../tests/testsData/results/res23.csv | 8 +++ .../tests/testsData/results/res24.csv | 7 +++ .../tests/testsData/results/res25.csv | 1 + .../tests/testsData/results/res26.csv | 11 ++++ .../tests/testsData/results/res27.csv | 8 +++ .../tests/testsData/results/res28.csv | 8 +++ .../tests/testsData/results/res29.csv | 2 + .../tests/testsData/rules/rule23.csv | 6 ++ .../tests/testsData/rules/rule24.csv | 5 ++ .../tests/testsData/rules/rule25.csv | 2 + .../tests/testsData/rules/rule26.csv | 6 ++ .../tests/testsData/rules/rule27.csv | 3 + .../tests/testsData/rules/rule28.csv | 5 ++ .../tests/testsData/rules/rule29.csv | 3 + 24 files changed, 325 insertions(+) create mode 100644 cmake/tests.cmake create mode 100755 modules/listDetector/tests/test.sh create mode 100644 modules/listDetector/tests/testsData/inputs/input23.csv create mode 100644 modules/listDetector/tests/testsData/inputs/input24.csv create mode 100644 modules/listDetector/tests/testsData/inputs/input25.csv create mode 100644 modules/listDetector/tests/testsData/inputs/input26.csv create mode 100644 modules/listDetector/tests/testsData/inputs/input27.csv create mode 100644 modules/listDetector/tests/testsData/inputs/input28.csv create mode 100644 modules/listDetector/tests/testsData/inputs/input29.csv create mode 100644 modules/listDetector/tests/testsData/results/res23.csv create mode 100644 modules/listDetector/tests/testsData/results/res24.csv create mode 100644 modules/listDetector/tests/testsData/results/res25.csv create mode 100644 modules/listDetector/tests/testsData/results/res26.csv create mode 100644 modules/listDetector/tests/testsData/results/res27.csv create mode 100644 modules/listDetector/tests/testsData/results/res28.csv create mode 100644 modules/listDetector/tests/testsData/results/res29.csv create mode 100644 modules/listDetector/tests/testsData/rules/rule23.csv create mode 100644 modules/listDetector/tests/testsData/rules/rule24.csv create mode 100644 modules/listDetector/tests/testsData/rules/rule25.csv create mode 100644 modules/listDetector/tests/testsData/rules/rule26.csv create mode 100755 modules/listDetector/tests/testsData/rules/rule27.csv create mode 100644 modules/listDetector/tests/testsData/rules/rule28.csv create mode 100644 modules/listDetector/tests/testsData/rules/rule29.csv diff --git a/CMakeLists.txt b/CMakeLists.txt index e35c9b76..6d0ad6a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(NM_NG_ENABLE_DOC_DOXYGEN "Enable build of code documentation" OFF) option(NM_NG_BUILD_WITH_ASAN "Build with Address Sanitizer (only for CMAKE_BUILD_TYPE=Debug)" OFF) option(NM_NG_BUILD_WITH_UBSAN "Build with Undefined Behavior Sanitizer (only for CMAKE_BUILD_TYPE=Debug)" OFF) +option(NM_NG_ENABLE_TESTS "Build with tests of modules" OFF) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic -Wall -Wextra -Wunused -Wconversion -Wsign-conversion") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -Werror") @@ -41,3 +42,7 @@ add_subdirectory(modules) add_subdirectory(common) add_subdirectory(pkg) add_subdirectory(doc) + +if (NM_NG_ENABLE_TESTS) + include(cmake/tests.cmake) +endif() diff --git a/cmake/tests.cmake b/cmake/tests.cmake new file mode 100644 index 00000000..9160216e --- /dev/null +++ b/cmake/tests.cmake @@ -0,0 +1,27 @@ +# File searches for test.sh script in each module subdirectory and runs it if present + +file(GLOB MODULE_DIRS RELATIVE ${CMAKE_SOURCE_DIR}/modules ${CMAKE_SOURCE_DIR}/modules/*) +enable_testing() + +add_custom_target(tests + COMMAND ctest --output-on-failure + VERBATIM +) + +foreach(MODULE ${MODULE_DIRS}) + if (NOT IS_DIRECTORY ${CMAKE_SOURCE_DIR}/modules/${MODULE}) + continue() + endif() + set(TEST_SCRIPT "${CMAKE_SOURCE_DIR}/modules/${MODULE}/tests/test.sh") + + if (EXISTS ${TEST_SCRIPT}) + add_test(NAME Test${MODULE} COMMAND bash ${TEST_SCRIPT} ${CMAKE_BINARY_DIR}/modules/${MODULE}/src/${MODULE}) + add_dependencies(tests ${MODULE}) + else() + add_custom_target(print_missing_test_for_${MODULE} + COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --yellow --bold "No test.sh found for: ${MODULE}. Skipping..." + VERBATIM + ) + add_dependencies(tests print_missing_test_for_${MODULE}) + endif() +endforeach() diff --git a/modules/listDetector/tests/test.sh b/modules/listDetector/tests/test.sh new file mode 100755 index 00000000..82a64523 --- /dev/null +++ b/modules/listDetector/tests/test.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +function exit_with_error { + pkill logger + pkill logreplay + pkill listDetector + exit 1 +} + +function process_started { + pid=$1 + if ! ps -p $pid > /dev/null + then + echo "Failed to start process" + exit_with_error + fi +} + +data_path="$(dirname "$0")/testsData/" +list_detector=$1 + +set -e +trap 'echo "Command \"$BASH_COMMAND\" failed!"; exit_with_error' ERR +for input_file in $data_path/inputs/*; do + index=$(echo "$input_file" | grep -o '[0-9]\+') + + res_file="/tmp/res" + logger -i "u:listDetector" -w $res_file & + logger_pid=$! + sleep 0.1 + + process_started $logger_pid + + $list_detector \ + -i "u:lr,u:listDetector" \ + -r "$data_path/rules/rule$index.csv" \ + -lm blacklist & + + detector_pid=$! + sleep 0.1 + process_started $detector_pid + + logreplay -i "u:lr" -f "$data_path/inputs/input$index.csv" 2>/dev/null & + sleep 0.1 + process_started $! + + wait $logger_pid + wait $detector_pid + + if [ -f "$res_file" ]; then + if ! cmp -s "$data_path/results/res$index.csv" "$res_file"; then + echo "Files results/res$index.csv and $res_file are not equal" + exit_with_error + fi + else + echo "File $res_file not found" + exit_with_error + fi +done + +echo "All tests passed" +exit 0 diff --git a/modules/listDetector/tests/testsData/inputs/input23.csv b/modules/listDetector/tests/testsData/inputs/input23.csv new file mode 100644 index 00000000..10e85a54 --- /dev/null +++ b/modules/listDetector/tests/testsData/inputs/input23.csv @@ -0,0 +1,20 @@ +ipaddr DST_IP, ipaddr SRC_IP, uint64 BYTES, string STR_ONE, string STR_TWO, uint32 PACKETS +54.175.219.8,54.175.219.8,123,test1,test1,123 +54.175.219.24,54.175.29.123,123,rterttest1,trtrttest11,123 +54.175.219.77,54.175.21.66,123,test123232,test132322,123 +54.175.219.99,54.175.3.17,123,test1222,test11113,123 +---------------------------------------------------------- +54.175.219.99,54.175.3.17,123,tes1222,tst11114,123 +54.175.219.99,54.175.3.17,123,test,test5,123 +54.175.219.99,54.175.3.17,123,t,test11116,123 +=========================================================== +54.175.210.19,54.175.219.16,133,test1,test2,124 +54.175.210.19,54.175.219.16,133,test1,test234234,124 +54.175.210.19,54.175.219.16,133,test1,test2123123123,124 +54.175.210.19,54.175.219.16,133,test1231231,test2,124 +----------------------------------------------------------- +54.175.21.19,54.175.219.16,133,test231231,test2,124 +54.175.210.19,54.175.219.16,133,tst1231231,test2,124 +54.176.210.19,54.175.219.16,133,test,test2,124 +54.175.210.19,55.175.219.11,133,test1231231,test,124 +======================================================== diff --git a/modules/listDetector/tests/testsData/inputs/input24.csv b/modules/listDetector/tests/testsData/inputs/input24.csv new file mode 100644 index 00000000..23cdaa38 --- /dev/null +++ b/modules/listDetector/tests/testsData/inputs/input24.csv @@ -0,0 +1,18 @@ +uint16 PORT, string STR1, string STR2, uint16 GARBAGE +1,match1,match2,3 +1,match12,match2,3 +1,match1,match22,3 +1,match2341,match2432,3 + +2,match1,match2,3 +2,match21,ma2tch2,3 + +3,match1,match2,3 +3,match12,match2,3 +3,match1,match22,3 +3,match12,match22,3 + +4,match1,match2,3 +4,match12,match2,3 +4,match1,match22,3 +4,match121,matc3h22,3 diff --git a/modules/listDetector/tests/testsData/inputs/input25.csv b/modules/listDetector/tests/testsData/inputs/input25.csv new file mode 100644 index 00000000..91944de1 --- /dev/null +++ b/modules/listDetector/tests/testsData/inputs/input25.csv @@ -0,0 +1,3 @@ +uint16 PORT, string STR1, string STR2, uint16 GARBAGE +3,match1,match2,3 +3,match1,match22,3 diff --git a/modules/listDetector/tests/testsData/inputs/input26.csv b/modules/listDetector/tests/testsData/inputs/input26.csv new file mode 100644 index 00000000..8948c7d6 --- /dev/null +++ b/modules/listDetector/tests/testsData/inputs/input26.csv @@ -0,0 +1,47 @@ +ipaddr IPA, ipaddr IPB, string STRA, string STRB, uint16 FIXEDA, uint8 FIXEDB, string STRC, ipaddr IPC +1.1.1.1,2.2.2.2,string1,string2,1234,56,string3,3.3.3.3 +--------------------------------------------------------- +1.1.1.2,2.2.2.2,string1,string2,1234,56,string3,3.3.3.3 +1.1.1.1,2.2.2.3,string1,string2,1234,56,string3,3.3.3.3 +1.1.1.1,2.2.2.2,string1,string2,1234,56,string3,4.3.3.3 +1.1.1.1,2.2.2.2,string,string2,1234,56,string3,3.3.3.3 +1.1.1.1,2.2.2.2,string1,stri,1234,56,string3,3.3.3.3 +1.1.1.1,2.2.2.2,string1,string2,1234,56,sng3,3.3.3.3 +1.1.1.1,2.2.2.2,string1,string2,1254,56,string3,3.3.3.3 +1.1.1.1,2.2.2.2,string1,string2,1234,66,string3,3.3.3.3 +============================================================ +1.1.2.1,2.2.3.2,string12,string22,1244,66,string32,3.3.4.3 +============================================================== +1.1.3.1,2.2.4.2,string1,string22,1254,76,string32,3.3.4.3 +1.1.3.255,2.2.4.255,string1,string22,1254,76,string32,3.3.4.2 +1.1.3.22,2.2.4.33,string1,string22,1254,76,string32,3.3.4.1 +---------------------------------------------------------- +1.1.13.1,2.2.4.2,string1,string22,1254,76,string32,3.3.4.3 +1.1.3.255,2.2.4.255,string1,string22,1254,76,string32,3.23.4.2 +1.1.3.22,2.2.4.33,string21,string22,1254,76,string32,3.3.4.1 +1.1.3.1,2.2.4.2,string1,string22,1254,86,string32,3.3.4.3 +1.1.3.255,2.2.4.255,string1,string22,1354,76,string32,3.3.4.2 +1.1.3.22,2.2.4.33,string1,string22,1254,76,stringg32,3.3.4.1 +1.1.3.22,2.2.4.33,string1,string22,1264,76,string32,3.3.4.1 +=============================================================== +1.1.4.1,2.2.5.2,string1,str2,1264,86,str,3.3.5.3 +1.1.4.255,2.2.5.244,string1,str23434,1264,86,kkkkkkstr,3.3.5.2 +1.1.4.100,2.2.5.200,string1,bgfdgdfstr2,1264,86,ssssssstr,3.3.5.1 +-------------------------------------------------------------------- +1.1.6.1,2.2.5.2,string1,str2,1264,86,str,3.3.5.3 +1.1.4.255,2.2.5.244,string1,str23434,1264,86,kkkkkkstr,3.4.5.2 +1.1.4.100,2.2.5.200,string1,bgfdgdfstr2,1264,86,sssssssr,3.3.5.1 +1.1.4.1,2.2.5.2,string1,str,1264,86,str,3.3.5.3 +1.1.4.255,2.2.5.244,string1,str23434,124,6,kkkkkkstr,3.3.5.2 +1.1.4.100,2.2.5.200,string,bgfdgdfstr2,1264,86,ssssssstr,3.3.5.1 +================================================================= +1.1.5.1,2.2.6.2,strign,strign12,1264,86,stri12,3.3.6.3 +1.1.5.234,2.2.6.233,strign,strign66,1264,86,stri33,3.3.6.1 +1.1.5.15,2.2.6.20,strign,strign11,1264,86,stri55,3.3.6.2 +1.1.5.234,2.2.7.233,strign,strign66,1264,86,stri33,3.3.6.1 +1.1.5.15,2.2.6.20,strign,strign11,1264,86,stri55,3.3.7.2 +-------------------------------------------------------------- +1.1.5.1,2.2.6.2,strin,strign12,1264,86,stri12,3.3.6.3 +1.1.5.234,2.2.6.233,strign,strign,1264,86,stri33,3.3.6.1 +1.1.5.15,2.2.6.20,strign,strign11,1264,86,stri5,3.3.6.2 +1.1.6.1,2.2.6.2,strign,strign12,1264,86,stri12,3.3.6.3 diff --git a/modules/listDetector/tests/testsData/inputs/input27.csv b/modules/listDetector/tests/testsData/inputs/input27.csv new file mode 100644 index 00000000..5a8696fb --- /dev/null +++ b/modules/listDetector/tests/testsData/inputs/input27.csv @@ -0,0 +1,19 @@ +ipaddr DST_IP, ipaddr SRC_IP, uint64 BYTES, string STR_ONE, string STR_TWO, uint32 PACKETS +55.175.219.8,54.175.219.8,123,test1,test1,123 +55.175.219.24,54.175.29.123,123,rterttest1,trtrttest11,666 +55.175.219.77,54.175.21.66,123,test123232,test132322,999 +55.175.219.99,54.175.3.17,123,test1222,test11113,111 +---------------------------------------------------------- +55.175.219.99,54.175.3.17,123,tes1222,tst11114,123 +55.175.219.99,54.175.3.17,123,test,test5,123 +55.175.219.99,54.175.3.17,123,t,test11116,123 +=========================================================== +54.175.210.169,54.175.219.16,133,test1,test2,124 +54.175.20.19,54.175.219.16,133,test1,test234234,124 +54.75.210.19,54.175.219.16,133,test1,test2123123123,124 +5.175.210.19,54.175.219.16,133,test1231231,test2,124 +----------------------------------------------------------- +54.175.210.19,55.175.219.16,133,tst1231231,test2,124 +54.176.210.19,55.175.219.16,133,test,test2,124 +54.175.210.19,55.175.219.11,133,test1231231,test,124 +======================================================== diff --git a/modules/listDetector/tests/testsData/inputs/input28.csv b/modules/listDetector/tests/testsData/inputs/input28.csv new file mode 100644 index 00000000..0087c860 --- /dev/null +++ b/modules/listDetector/tests/testsData/inputs/input28.csv @@ -0,0 +1,41 @@ +ipaddr IPA, ipaddr IPB, string STRA, string STRB, uint16 FIXEDA, uint8 FIXEDB, string STRC, ipaddr IPC +1.1.1.1,2.2.2.2,string1,string2,1234,56,string3,3.3.3.3 +--------------------------------------------------------- +1.1.1.2,2.2.2.2,string1,string2,1234,56,string3,3.3.3.3 +1.1.1.1,2.2.2.3,string1,string2,1234,56,string3,3.3.3.3 +1.1.1.1,2.2.2.2,string,string2,1234,56,string3,3.3.3.3 +1.1.1.1,2.2.2.2,string1,stri,1234,56,string3,3.3.3.3 +1.1.1.1,2.2.2.2,string1,string2,1234,56,sng3,3.3.3.3 +1.1.1.1,2.2.2.2,string1,string2,1254,56,string3,3.3.3.3 +1.1.1.1,2.2.2.2,string1,string2,1234,66,string3,3.3.3.3 +============================================================ +1.1.2.1,2.2.3.2,string12,string22,1244,66,string32,3.3.4.3 +============================================================== + +---------------------------------------------------------- +1.1.13.1,2.2.4.2,string1,string22,1254,76,string32,3.3.4.3 +1.1.3.255,2.2.4.255,string1,string22,1254,76,string32,3.23.4.2 +1.1.3.1,2.2.4.2,string1,string22,1254,86,string32,3.3.4.3 +1.1.3.255,2.2.4.255,string1,string22,1354,76,string32,3.3.4.2 +1.1.3.22,2.2.4.33,string1,string22,1254,76,stringg32,3.3.4.1 +1.1.3.22,2.2.4.33,string1,string22,1264,76,string32,3.3.4.1 +=============================================================== +1.1.4.1,2.2.5.2,string1,str2,1264,868,str,3.3.5.3 +1.1.4.255,2.2.5.244,string1,str23434,1264,816,kkkkkkstr,3.3.5.2 +1.1.4.100,2.2.5.200,string1,bgfdgdfstr2,1264,986,ssssssstr,3.3.5.1 +-------------------------------------------------------------------- +1.1.6.1,2.2.5.2,string1,str2,1264,86,str,3.3.5.3 +1.1.4.255,2.2.5.244,string1,str23434,1264,86,kkkkkkstr,3.4.5.2 +1.1.4.100,2.2.5.200,string1,bgfdgdfstr2,1264,86,sssssssr,3.3.5.1 +1.1.4.1,2.2.5.2,string1,str,1264,86,str,3.3.5.3 +1.1.4.100,2.2.5.200,string,bgfdgdfstr2,1264,86,ssssssstr,3.3.5.1 +================================================================= +1.1.5.1,2.2.6.2,strign,strign12,1264,86,stri12,3.3.6.3 +1.1.5.234,2.2.6.233,strign,strign66,1264,86,stri33,3.3.6.1 +1.1.5.15,2.2.6.20,strign,strign11,1264,86,stri55,3.3.6.2 +-------------------------------------------------------------- +1.1.5.234,2.2.7.233,strign,strign66,1264,86,stri33,3.3.6.1 +1.1.5.1,2.2.6.2,strin,strign12,1264,86,stri12,3.3.6.3 +1.1.5.234,2.2.6.233,strign,strign,1264,86,stri33,3.3.6.1 +1.1.5.15,2.2.6.20,strign,strign11,1264,86,stri5,3.3.6.2 +1.1.6.1,2.2.6.2,strign,strign12,1264,86,stri12,3.3.6.3 diff --git a/modules/listDetector/tests/testsData/inputs/input29.csv b/modules/listDetector/tests/testsData/inputs/input29.csv new file mode 100644 index 00000000..0b6b59a6 --- /dev/null +++ b/modules/listDetector/tests/testsData/inputs/input29.csv @@ -0,0 +1,8 @@ +ipaddr IPA, ipaddr IPB, ipaddr IPC +2001:0000:130F:0000:0000:09C0:876A:130A,2001:0000:130F:0000:0000:09C0:876A:130A,2001:0000:130F:0000:0000:09C0:876A:1301 +================================================================================================================================ +2101:0000:130F:0000:0000:09C0:876A:000B,2101:0000:130F:0000:0000:09C0:876A:100B,2101:0000:130F:0000:0000:09C0:876A:150B +---------------------------------------------------------------------------------------------------------------------- +2101:1000:130F:0000:0000:09C0:876A:000B,2101:0000:130F:0000:0000:09C0:876A:100B,2101:0000:130F:0000:0000:09C0:876A:150B +2101:0000:130F:0000:0000:09C0:876A:000B,2101:0000:130F:0000:0000:09C0:876A:100B,2101:0000:330F:0000:0000:09C0:876A:150B +2101:0000:130F:0000:0000:09C0:876A:000B,4101:0000:130F:0000:0000:09C0:876A:100B,2101:0000:130F:0000:0000:09C0:876A:150B diff --git a/modules/listDetector/tests/testsData/results/res23.csv b/modules/listDetector/tests/testsData/results/res23.csv new file mode 100644 index 00000000..678904f4 --- /dev/null +++ b/modules/listDetector/tests/testsData/results/res23.csv @@ -0,0 +1,8 @@ +54.175.219.8,54.175.219.8,123,123,"test1","test1" +54.175.219.24,54.175.29.123,123,123,"rterttest1","trtrttest11" +54.175.219.77,54.175.21.66,123,123,"test123232","test132322" +54.175.219.99,54.175.3.17,123,123,"test1222","test11113" +54.175.210.19,54.175.219.16,133,124,"test1","test2" +54.175.210.19,54.175.219.16,133,124,"test1","test234234" +54.175.210.19,54.175.219.16,133,124,"test1","test2123123123" +54.175.210.19,54.175.219.16,133,124,"test1231231","test2" diff --git a/modules/listDetector/tests/testsData/results/res24.csv b/modules/listDetector/tests/testsData/results/res24.csv new file mode 100644 index 00000000..e89aa816 --- /dev/null +++ b/modules/listDetector/tests/testsData/results/res24.csv @@ -0,0 +1,7 @@ +3,1,"match1","match2" +3,2,"match1","match2" +3,3,"match1","match2" +3,3,"match1","match22" +3,4,"match1","match2" +3,4,"match12","match2" +3,4,"match1","match22" diff --git a/modules/listDetector/tests/testsData/results/res25.csv b/modules/listDetector/tests/testsData/results/res25.csv new file mode 100644 index 00000000..56fbb4fe --- /dev/null +++ b/modules/listDetector/tests/testsData/results/res25.csv @@ -0,0 +1 @@ +3,3,"match1","match2" diff --git a/modules/listDetector/tests/testsData/results/res26.csv b/modules/listDetector/tests/testsData/results/res26.csv new file mode 100644 index 00000000..d8272875 --- /dev/null +++ b/modules/listDetector/tests/testsData/results/res26.csv @@ -0,0 +1,11 @@ +1.1.1.1,2.2.2.2,3.3.3.3,1234,56,"string1","string2","string3" +1.1.2.1,2.2.3.2,3.3.4.3,1244,66,"string12","string22","string32" +1.1.3.1,2.2.4.2,3.3.4.3,1254,76,"string1","string22","string32" +1.1.3.255,2.2.4.255,3.3.4.2,1254,76,"string1","string22","string32" +1.1.3.22,2.2.4.33,3.3.4.1,1254,76,"string1","string22","string32" +1.1.4.1,2.2.5.2,3.3.5.3,1264,86,"string1","str2","str" +1.1.4.255,2.2.5.244,3.3.5.2,1264,86,"string1","str23434","kkkkkkstr" +1.1.4.100,2.2.5.200,3.3.5.1,1264,86,"string1","bgfdgdfstr2","ssssssstr" +1.1.5.1,2.2.6.2,3.3.6.3,1264,86,"strign","strign12","stri12" +1.1.5.234,2.2.6.233,3.3.6.1,1264,86,"strign","strign66","stri33" +1.1.5.15,2.2.6.20,3.3.6.2,1264,86,"strign","strign11","stri55" diff --git a/modules/listDetector/tests/testsData/results/res27.csv b/modules/listDetector/tests/testsData/results/res27.csv new file mode 100644 index 00000000..9192d55d --- /dev/null +++ b/modules/listDetector/tests/testsData/results/res27.csv @@ -0,0 +1,8 @@ +55.175.219.8,54.175.219.8,123,123,"test1","test1" +55.175.219.24,54.175.29.123,123,666,"rterttest1","trtrttest11" +55.175.219.77,54.175.21.66,123,999,"test123232","test132322" +55.175.219.99,54.175.3.17,123,111,"test1222","test11113" +54.175.210.169,54.175.219.16,133,124,"test1","test2" +54.175.20.19,54.175.219.16,133,124,"test1","test234234" +54.75.210.19,54.175.219.16,133,124,"test1","test2123123123" +5.175.210.19,54.175.219.16,133,124,"test1231231","test2" diff --git a/modules/listDetector/tests/testsData/results/res28.csv b/modules/listDetector/tests/testsData/results/res28.csv new file mode 100644 index 00000000..f8016574 --- /dev/null +++ b/modules/listDetector/tests/testsData/results/res28.csv @@ -0,0 +1,8 @@ +1.1.1.1,2.2.2.2,3.3.3.3,1234,56,"string1","string2","string3" +1.1.2.1,2.2.3.2,3.3.4.3,1244,66,"string12","string22","string32" +1.1.4.1,2.2.5.2,3.3.5.3,1264,100,"string1","str2","str" +1.1.4.255,2.2.5.244,3.3.5.2,1264,48,"string1","str23434","kkkkkkstr" +1.1.4.100,2.2.5.200,3.3.5.1,1264,218,"string1","bgfdgdfstr2","ssssssstr" +1.1.5.1,2.2.6.2,3.3.6.3,1264,86,"strign","strign12","stri12" +1.1.5.234,2.2.6.233,3.3.6.1,1264,86,"strign","strign66","stri33" +1.1.5.15,2.2.6.20,3.3.6.2,1264,86,"strign","strign11","stri55" diff --git a/modules/listDetector/tests/testsData/results/res29.csv b/modules/listDetector/tests/testsData/results/res29.csv new file mode 100644 index 00000000..3797ede2 --- /dev/null +++ b/modules/listDetector/tests/testsData/results/res29.csv @@ -0,0 +1,2 @@ +2001:0:130f::9c0:876a:130a,2001:0:130f::9c0:876a:130a,2001:0:130f::9c0:876a:1301 +2101:0:130f::9c0:876a:b,2101:0:130f::9c0:876a:100b,2101:0:130f::9c0:876a:150b diff --git a/modules/listDetector/tests/testsData/rules/rule23.csv b/modules/listDetector/tests/testsData/rules/rule23.csv new file mode 100644 index 00000000..57a2ec4a --- /dev/null +++ b/modules/listDetector/tests/testsData/rules/rule23.csv @@ -0,0 +1,6 @@ +ipaddr DST_IP, ipaddr SRC_IP, uint64 BYTES, string STR_ONE, string STR_TWO, uint32 PACKETS +54.175.219.8/8,54.175.219.8/16,123,R"(.*test1.*)",R"(.*test1.*)",123 +54.175.210.8/24,54.175.219.8/12,133,R"(.*test1.*)",R"(.*test2.*)",124 +54.175.2.8/8,54.1.219.8/24,43,R"(.*test3.*)",R"(.*test3.*)",125 +54.175.19.8/16,5.175.219.8/20,153,R"(.*test2.*)",R"(.*test4.*)",126 +54.175.9.8/24,4.175.219.8/5,163,R"(.*test3.*)",R"(.*test2.*)",127 diff --git a/modules/listDetector/tests/testsData/rules/rule24.csv b/modules/listDetector/tests/testsData/rules/rule24.csv new file mode 100644 index 00000000..29b34a73 --- /dev/null +++ b/modules/listDetector/tests/testsData/rules/rule24.csv @@ -0,0 +1,5 @@ +uint16 PORT, string STR1, string STR2 +1,match1,match2 +2,R"(match.)",match2 +3,match1,R"(match.)" +4,R"(match.)",R"(match.)" diff --git a/modules/listDetector/tests/testsData/rules/rule25.csv b/modules/listDetector/tests/testsData/rules/rule25.csv new file mode 100644 index 00000000..84d2ebf4 --- /dev/null +++ b/modules/listDetector/tests/testsData/rules/rule25.csv @@ -0,0 +1,2 @@ +uint16 PORT, string STR1, string STR2 +3,match1,R"(match.$)" diff --git a/modules/listDetector/tests/testsData/rules/rule26.csv b/modules/listDetector/tests/testsData/rules/rule26.csv new file mode 100644 index 00000000..bcb89e50 --- /dev/null +++ b/modules/listDetector/tests/testsData/rules/rule26.csv @@ -0,0 +1,6 @@ +ipaddr IPA, ipaddr IPB, string STRA, string STRB, uint16 FIXEDA, uint8 FIXEDB, string STRC, ipaddr IPC +1.1.1.1,2.2.2.2,string1,string2,1234,56,string3,3.3.3.3 +1.1.2.1,2.2.3.2,string12,string22,1244,66,string32,3.3.4.3 +1.1.3.1/24,2.2.4.2/24,R"(string1)",string22,1254,76,string32,3.3.4.3/30 +1.1.4.1/24,2.2.5.2/24,string1,R"(str2)",1264,86,R"(str)",3.3.5.3/30 +1.1.5.1/24,2.2.6.2/24,R"(strign)",R"(strign..)",1264,86,R"(stri..)",3.3.6.3/30 diff --git a/modules/listDetector/tests/testsData/rules/rule27.csv b/modules/listDetector/tests/testsData/rules/rule27.csv new file mode 100755 index 00000000..eb6b8acb --- /dev/null +++ b/modules/listDetector/tests/testsData/rules/rule27.csv @@ -0,0 +1,3 @@ +ipaddr DST_IP, ipaddr SRC_IP, uint64 BYTES, string STR_ONE, string STR_TWO, uint32 PACKETS +55.175.219.8/8,54.175.219.8/16,123,R"(.*test1.*)",R"(.*test1.*)", +,54.175.219.8/12,133,R"(.*test1.*)",R"(.*test2.*)",124 diff --git a/modules/listDetector/tests/testsData/rules/rule28.csv b/modules/listDetector/tests/testsData/rules/rule28.csv new file mode 100644 index 00000000..77f594e9 --- /dev/null +++ b/modules/listDetector/tests/testsData/rules/rule28.csv @@ -0,0 +1,5 @@ +ipaddr IPA, ipaddr IPB, string STRA, string STRB, uint16 FIXEDA, uint8 FIXEDB, string STRC, ipaddr IPC +1.1.1.1,2.2.2.2,string1,string2,1234,56,string3, +1.1.2.1,2.2.3.2,string12,string22,1244,,string32,3.3.4.3 +1.1.4.1/24,2.2.5.2/24,string1,R"(str2)",1264,,R"(str)",3.3.5.3/30 +1.1.5.1/24,2.2.6.2/24,R"(strign)",R"(strign..)",1264,86,R"(stri..)", diff --git a/modules/listDetector/tests/testsData/rules/rule29.csv b/modules/listDetector/tests/testsData/rules/rule29.csv new file mode 100644 index 00000000..6aa30f94 --- /dev/null +++ b/modules/listDetector/tests/testsData/rules/rule29.csv @@ -0,0 +1,3 @@ +ipaddr IPA, ipaddr IPB, ipaddr IPC +2001:0000:130F:0000:0000:09C0:876A:130B/120, 2001:0000:130F:0000:0000:09C0:876A:130B/120, 2001:0000:130F:0000:0000:09C0:876A:130B/120 +2101:0000:130F:0000:0000:09C0:876A:130B/100, 2101:0000:130F:0000:0000:09C0:876A:130B/100, 2101:0000:130F:0000:0000:09C0:876A:130B/100 From caef81b235c4713b2f7f490ec9d0bb2e1004547c Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sun, 9 Mar 2025 20:01:26 +0100 Subject: [PATCH 15/15] List Detector - Add tests to CI --- .../actions/install-dependencies/action.yml | 2 +- .github/workflows/ci.yml | 5 +++ .github/workflows/tests.yml | 37 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/actions/install-dependencies/action.yml b/.github/actions/install-dependencies/action.yml index 3004c0ab..0ede74d6 100644 --- a/.github/actions/install-dependencies/action.yml +++ b/.github/actions/install-dependencies/action.yml @@ -29,4 +29,4 @@ runs: if: ${{ inputs.clang-tools == 'true' }} shell: bash run: | - dnf install -y clang clang-tools-extra + dnf install -y clang clang-tools-extra \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66845676..c98686bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,11 @@ jobs: uses: ./.github/workflows/build.yml with: os: ${{ matrix.os }} + tests: + needs: [build-os-matrix, build] + uses: ./.github/workflows/tests.yml + with: + os: "oraclelinux:9" rpm-install: needs: [build-os-matrix, build] strategy: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..ec6a7142 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,37 @@ +name: tests + +on: + workflow_call: + inputs: + os: + required: true + type: string + +jobs: + tests: + runs-on: ubuntu-latest + container: ${{ inputs.os }} + steps: + - name: Install git + run: dnf install -y git + - name: Check out repository code + uses: actions/checkout@v4 + - name: Install dependencies + uses: ./.github/actions/install-dependencies + - name: Install nemea + run: | + dnf copr enable @CESNET/NEMEA-testing + dnf copr enable @CESNET/NEMEA + dnf install -y epel-release + dnf install -y nemea-framework-devel + dnf install -y nemea + dnf install -y procps-ng autoconf + echo PATH=/usr/bin/nemea:$PATH >> $GITHUB_ENV + - name: Compile modules + run: | + cmake -S . -B build -DNM_NG_ENABLE_TESTS=On + make -C build install + - name: Run tests + run: | + echo "Path=$PATH" + make -C build tests \ No newline at end of file