From 2e137398c336de7f477d0b729419abe70d0a2c0b Mon Sep 17 00:00:00 2001 From: Himanshu Pal Date: Mon, 21 Apr 2025 15:09:14 +0530 Subject: [PATCH 1/3] poc for azure observability --- azure-observability/azure_resources.tf | 228 +++++++++++++++++++++++++ azure-observability/local.tf | 11 ++ azure-observability/output.tf | 3 + azure-observability/providers.tf | 35 ++++ azure-observability/sumo_resources.tf | 19 +++ azure-observability/variables.tf | 131 ++++++++++++++ azure-observability/versions.tf | 25 +++ 7 files changed, 452 insertions(+) create mode 100644 azure-observability/azure_resources.tf create mode 100644 azure-observability/local.tf create mode 100644 azure-observability/output.tf create mode 100644 azure-observability/providers.tf create mode 100644 azure-observability/sumo_resources.tf create mode 100644 azure-observability/variables.tf create mode 100644 azure-observability/versions.tf diff --git a/azure-observability/azure_resources.tf b/azure-observability/azure_resources.tf new file mode 100644 index 00000000..0e56be09 --- /dev/null +++ b/azure-observability/azure_resources.tf @@ -0,0 +1,228 @@ +# multiple location +# separate source module +# multiple namespace +# # Todo Create a policy + +# Create a Resource Group +resource "azurerm_resource_group" "rg" { + name = var.resource_group_name + location = var.location +} + +# Create an Event Hub Namespace +resource "azurerm_eventhub_namespace" "namespace" { + name = var.eventhub_namespace_name + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + sku = "Standard" + capacity = var.throughput_units + + # Todo take tags from vars file and iterate over multiple tags + tags = { + version = local.solution_version + } +} + +# Create an Event Hub +resource "azurerm_eventhub" "eventhub" { + name = var.eventhub_name + namespace_id = azurerm_eventhub_namespace.namespace.id + + # 1 partition = 1 MB/sec + partition_count = 4 + message_retention = 7 +} + +# Create a Shared Access Policy with listen permissions +resource "azurerm_eventhub_namespace_authorization_rule" "sumo_collection_policy" { + name = var.policy_name + namespace_name = azurerm_eventhub_namespace.namespace.name + resource_group_name = azurerm_resource_group.rg.name + listen = true + send = false + manage = false +} + +# Sumo Collection sources +resource "sumologic_collector" "sumo_collector" { + # Todo collector name from the variable + # Todo add tenant subscription in collector to uniquely identify + name = "Azure Observability Collector" + description = "created via terraform" + + # name = local.collector_name + # description = var.sumologic_collector_details.description + # fields = var.sumologic_collector_details.fields + # timezone = "UTC" + fields = { + tenant_name = "azure_account" + } +} + +resource "sumologic_azure_event_hub_log_source" "sumo_azure_event_hub_log_source" { + # Todo separate + name = "Azure Logs Source" + description = "created via terraform uses ${var.eventhub_name}" + category = "azure/eventhub/logs" + content_type = "AzureEventHubLog" + collector_id = "${sumologic_collector.sumo_collector.id}" + depends_on = [azurerm_eventhub.eventhub] + authentication { + type = "AzureEventHubAuthentication" + shared_access_policy_name = var.policy_name + shared_access_policy_key = azurerm_eventhub_namespace_authorization_rule.sumo_collection_policy.primary_key + } + path { + type = "AzureEventHubPath" + namespace = azurerm_eventhub_namespace.namespace.name + event_hub_name = var.eventhub_name + consumer_group = "$Default" + # Todo test for US Gov, take it as separate variable + region = "Commercial" + } +} + +data "azurerm_client_config" "current" {} + +resource "sumologic_azure_metrics_source" "sumo_azure_event_hub_metrics_source" { + # one source for all regions + # Todo take region as input + # Todo take namespaces as input + name = "Azure Metrics Source" + description = "created via terraform uses Azure Monitor API" + category = "azure/eventhub/metrics" + content_type = "AzureMetrics" + collector_id = "${sumologic_collector.sumo_collector.id}" + authentication { + type = "AzureClientSecretAuthentication" + tenant_id = data.azurerm_client_config.current.tenant_id + # Todo create client id and client secret with contributors role and use service using script and use the same in providers and below + # https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/service_principal_client_secret + client_id = var.azure_client_id + client_secret = var.azure_client_secret + } + path { + type = "AzureMetricsPath" + limit_to_namespaces = ["Microsoft.Web/sites"] + } + lifecycle { + ignore_changes = [authentication.0.client_secret] + } +} + + +data "azurerm_eventhub_namespace_authorization_rule" "default_rule" { + name = "RootManageSharedAccessKey" + namespace_name = var.eventhub_namespace_name + resource_group_name = azurerm_resource_group.rg.name + depends_on = [azurerm_eventhub_namespace.namespace] +} + +# Todo for each for each target resource +data "azurerm_monitor_diagnostic_categories" "function_app_category" { + # Todo take in variable + resource_id = var.target_resource_ids[0] +} + +# Todo Create a autosubscribe function to create diagnostic settings +# Create a Diagnostic Setting for Function App logs +resource "azurerm_monitor_diagnostic_setting" "diagnostic_setting_logs" { + name = "sumo_export_logs" + eventhub_name = var.eventhub_name + target_resource_id = var.target_resource_ids[0] + eventhub_authorization_rule_id = data.azurerm_eventhub_namespace_authorization_rule.default_rule.id + + # Select the resource from the list https://learn.microsoft.com/en-gb/azure/azure-monitor/platform/resource-logs-schema#service-specific-schemas + # Select the category from the link opened from above link for example for function app it opens this link - https://learn.microsoft.com/en-us/azure/azure-functions/monitor-functions-reference?tabs=consumption-plan#resource-logs + + + dynamic "enabled_log" { + for_each = data.azurerm_monitor_diagnostic_categories.function_app_category.log_category_types + + content { + category = enabled_log.value + } + } + + + metric { + category = "AllMetrics" + enabled = false + + retention_policy { + days = 0 + enabled = false + } + } + + # dynamic log { + # for_each = sort(data.azurerm_monitor_diagnostic_categories.default.logs) + # content { + # category = log.value + # enabled = true + + # retention_policy { + # enabled = true + # days = 30 + # } + # } + # } + + # # this needs to be here with enabled = false to prevent TF from showing changes happening with each plan/apply + # dynamic metric { + # for_each = sort(data.azurerm_monitor_diagnostic_categories.default.metrics) + # content { + # category = metric.value + # enabled = false + + # retention_policy { + # enabled = false + # days = 0 + # } + # } + # } + + # metric { + # category = "AllMetrics" + # } + + # logs { + # category = "Function Application Logs" + # enabled = true + # retention_policy { + # enabled = false + # } + # } + + # metrics { + # category = "AllMetrics" + # enabled = true + # retention_policy { + # enabled = false + # } + # } +} + +# data "azurerm_storage_account" "main" { +# name = "allbloblogseastus" +# resource_group_name = azurerm_resource_group.rg.name +# } + +# resource "azurerm_monitor_diagnostic_setting" "main" { +# name = "sumo_export_logs" +# eventhub_name = var.eventhub_name +# target_resource_id = "${data.azurerm_storage_account.main.id}/blobServices/default" +# eventhub_authorization_rule_id = data.azurerm_eventhub_namespace_authorization_rule.default_rule.id +# enabled_log { +# category = "StorageRead" +# } + +# enabled_log { +# category = "StorageWrite" +# } + +# enabled_log { +# category = "StorageDelete" +# } +# } + diff --git a/azure-observability/local.tf b/azure-observability/local.tf new file mode 100644 index 00000000..801dac79 --- /dev/null +++ b/azure-observability/local.tf @@ -0,0 +1,11 @@ +locals { + sumologic_service_endpoint = var.sumologic_environment == "us1" ? "https://service.sumologic.com" : (contains(["stag", "long"], var.sumologic_environment) ? "https://${var.sumologic_environment}.sumologic.net" : "https://service.${var.sumologic_environment}.sumologic.com") + sumologic_api_endpoint = var.sumologic_environment == "us1" ? "https://api.sumologic.com/api" : (contains(["stag", "long"], var.sumologic_environment) ? "https://${var.sumologic_environment}-api.sumologic.net/api" : "https://api.${var.sumologic_environment}.sumologic.com/api") + + # is_adminMode = var.apps_folder_installation_location == "Admin Recommended Folder" ? true : false + + apps_to_install = compact([for s in split(",", var.apps_names_to_install) : trimspace(s)]) + + solution_version = "v1.0.0" + +} diff --git a/azure-observability/output.tf b/azure-observability/output.tf new file mode 100644 index 00000000..37cdc23c --- /dev/null +++ b/azure-observability/output.tf @@ -0,0 +1,3 @@ +output "resource_group_name" { + value = azurerm_resource_group.rg.name +} \ No newline at end of file diff --git a/azure-observability/providers.tf b/azure-observability/providers.tf new file mode 100644 index 00000000..12739500 --- /dev/null +++ b/azure-observability/providers.tf @@ -0,0 +1,35 @@ +# provider "sumologic" { +# environment = var.sumologic_environment +# access_id = var.sumologic_access_id +# access_key = var.sumologic_access_key +# admin_mode = true +# alias = "admin" +# } + +provider "sumologic" { + environment = var.sumologic_environment + access_id = var.sumologic_access_id + access_key = var.sumologic_access_key + # base_url = local.sumologic_service_endpoint +} + +provider "azurerm" { + # The AzureRM Provider supports authenticating using via the Azure CLI, a Managed Identity + # and a Service Principal. More information on the authentication methods supported by + # the AzureRM Provider can be found here: + # https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs#authenticating-to-azure + # recommended authenticating using the Azure CLI when running Terraform locally. + + # The features block allows changing the behaviour of the Azure Provider, more + # information can be found here: + # https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/features-block + subscription_id = var.azure_subscription_id + features { + + resource_group { + prevent_deletion_if_contains_resources = true + } + + + } +} \ No newline at end of file diff --git a/azure-observability/sumo_resources.tf b/azure-observability/sumo_resources.tf new file mode 100644 index 00000000..a7c7bf48 --- /dev/null +++ b/azure-observability/sumo_resources.tf @@ -0,0 +1,19 @@ +# have to use version as hardcoded because during apply command latest always tries to update app resource and fails if the app with latest version is already installed +resource "sumologic_app" "azure_function_app" { + uuid = "a0fb1bf0-2ab4-4f69-bf7e-5d97a176c7ea" + version = "1.0.3" + # Todo namespace to app mapping + # separate app module + count = contains(local.apps_to_install, "Azure Functions") ? 1 : 0 +} + +resource "sumologic_app" "azure_web_app" { + uuid = "a4741497-31c6-4fb2-a236-0223e98b59e8" + version = "1.0.1" + count = contains(local.apps_to_install, "Azure Web Apps") ? 1 : 0 +} +resource "sumologic_app" "azure_cosmos_db_app" { + uuid = "d9ac4e28-13d6-4e69-8dcc-63fd6cb3bc80" + version = "1.0.1" + count = contains(local.apps_to_install, "Azure CosmosDB") ? 1 : 0 +} \ No newline at end of file diff --git a/azure-observability/variables.tf b/azure-observability/variables.tf new file mode 100644 index 00000000..b5e13c68 --- /dev/null +++ b/azure-observability/variables.tf @@ -0,0 +1,131 @@ + + +variable "azure_subscription_id" { + description = "The subscription id where your azure resources are present" + type = string +} + +variable "azure_client_id" { + description = "The client id " + type = string +} + +variable "azure_client_secret" { + description = "The client secret" + type = string +} + +variable target_resource_ids { + type = list + description = "List of target azure resources whose logs and metrics you want to collect in the provided region and subscription" + default = ["/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azurefunctiontrace/providers/Microsoft.Web/sites/azurefunctiontrace/"] +} + + + + +variable "resource_group_name" { + description = "The name of the Resource Group." + default = "hpalazureobservability" + type = string +} + +variable "eventhub_namespace_name" { + description = "The name of the Event Hub Namespace." + type = string + default = "sumologiclogseventhubnamespace" +} + + +variable "eventhub_name" { + description = "The name of the Event Hub." + type = string + default = "sumologiclogseventhub" +} + +variable "eventhub_metrics_name" { + description = "The name of the Event Hub." + type = string + default = "sumologicmetricseventhub" +} + +variable "location" { + description = "The location for the resources." + type = string + default = "East US" +} + +variable "throughput_units" { + description = "The number of throughput units for the Event Hub Namespace." + type = number + default = 5 +} + +variable "policy_name" { + description = "The name of the Shared Access Policy." + type = string + default = "SumoCollectionPolicy" +} + +variable "sumologic_environment" { + type = string + description = "Enter au, ca, de, eu, jp, us2, in, kr, fed or us1. For more information on Sumo Logic deployments visit https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and-Firewall-Security" + + validation { + condition = contains([ + "stag", + "long", + "au", + "ca", + "de", + "eu", + "jp", + "us1", + "us2", + "in", + "kr", + "fed"], var.sumologic_environment) + error_message = "The value must be one of au, ca, de, eu, jp, us1, us2, in, kr or fed." + } +} + +variable "sumologic_access_id" { + type = string + description = "Sumo Logic Access ID. Visit https://help.sumologic.com/Manage/Security/Access-Keys#Create_an_access_key" + + validation { + condition = can(regex("\\w+", var.sumologic_access_id)) + error_message = "The SumoLogic access ID must contain valid characters." + } +} + +variable "sumologic_access_key" { + type = string + description = "Sumo Logic Access Key. Visit https://help.sumologic.com/Manage/Security/Access-Keys#Create_an_access_key" + sensitive = true + + validation { + condition = can(regex("\\w+", var.sumologic_access_key)) + error_message = "The SumoLogic access key must contain valid characters." + } +} + + + +variable "apps_names_to_install" { + type = string + description = < Date: Mon, 21 Apr 2025 15:53:13 +0530 Subject: [PATCH 2/3] added azure github action for static tests and security tests --- .github/workflows/azo-tf-test.yml | 66 +++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/azo-tf-test.yml diff --git a/.github/workflows/azo-tf-test.yml b/.github/workflows/azo-tf-test.yml new file mode 100644 index 00000000..94293f08 --- /dev/null +++ b/.github/workflows/azo-tf-test.yml @@ -0,0 +1,66 @@ +name: "Azure TF template tests" +on: + pull_request: + paths: + - 'azure-observability-terraform/**' + +jobs: + + ValidateTF: + runs-on: ubuntu-latest + name: "Validatation (format & syntax)" + defaults: + run: + working-directory: ./azure-observability-terraform + steps: + - uses: actions/checkout@v4 + name: Checkout source code + + - uses: hashicorp/setup-terraform@v3 + name: Setup Terraform + # The default configuration installs the latest version of Terraform CLI + # with: + # terraform_version: "1.1.7" + + - name: Terraform fmt + id: fmt + run: terraform fmt -check -recursive -diff + continue-on-error: true + + - name: Terraform Init + id: init + run: terraform init + + - name: Terraform Validate + id: validate + run: terraform validate + + TFSecurityChecksUsingCheckov: + name: "Security Checks (checkov)" + runs-on: "ubuntu-latest" + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - uses: bridgecrewio/checkov-action@master + with: + directory: 'azure-observability-terraform/' + quiet: true + framework: terraform + output_format: cli + output_bc_ids: false + download_external_modules: true + # skip_check: + + TFSecurityChecksUsingTfsec: + name: "Security Checks (tfsec)" + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: tfsec + uses: aquasecurity/tfsec-action@v1.0.0 + with: + working_directory: 'azure-observability-terraform/' From 4193376c4c7d016704c6d5ad83b6d5357061d7a0 Mon Sep 17 00:00:00 2001 From: Himanshu Pal Date: Mon, 21 Apr 2025 17:13:10 +0530 Subject: [PATCH 3/3] updated folder name --- .github/workflows/azo-tf-test.yml | 8 ++++---- .../azure_resources.tf | 0 .../local.tf | 0 .../output.tf | 0 .../providers.tf | 0 .../sumo_resources.tf | 0 .../variables.tf | 0 .../versions.tf | 0 8 files changed, 4 insertions(+), 4 deletions(-) rename {azure-observability => azure-collection-terraform}/azure_resources.tf (100%) rename {azure-observability => azure-collection-terraform}/local.tf (100%) rename {azure-observability => azure-collection-terraform}/output.tf (100%) rename {azure-observability => azure-collection-terraform}/providers.tf (100%) rename {azure-observability => azure-collection-terraform}/sumo_resources.tf (100%) rename {azure-observability => azure-collection-terraform}/variables.tf (100%) rename {azure-observability => azure-collection-terraform}/versions.tf (100%) diff --git a/.github/workflows/azo-tf-test.yml b/.github/workflows/azo-tf-test.yml index 94293f08..55993487 100644 --- a/.github/workflows/azo-tf-test.yml +++ b/.github/workflows/azo-tf-test.yml @@ -2,7 +2,7 @@ name: "Azure TF template tests" on: pull_request: paths: - - 'azure-observability-terraform/**' + - 'azure-collection-terraform/**' jobs: @@ -11,7 +11,7 @@ jobs: name: "Validatation (format & syntax)" defaults: run: - working-directory: ./azure-observability-terraform + working-directory: ./azure-collection-terraform steps: - uses: actions/checkout@v4 name: Checkout source code @@ -44,7 +44,7 @@ jobs: - uses: bridgecrewio/checkov-action@master with: - directory: 'azure-observability-terraform/' + directory: 'azure-collection-terraform/' quiet: true framework: terraform output_format: cli @@ -63,4 +63,4 @@ jobs: - name: tfsec uses: aquasecurity/tfsec-action@v1.0.0 with: - working_directory: 'azure-observability-terraform/' + working_directory: 'azure-collection-terraform/' diff --git a/azure-observability/azure_resources.tf b/azure-collection-terraform/azure_resources.tf similarity index 100% rename from azure-observability/azure_resources.tf rename to azure-collection-terraform/azure_resources.tf diff --git a/azure-observability/local.tf b/azure-collection-terraform/local.tf similarity index 100% rename from azure-observability/local.tf rename to azure-collection-terraform/local.tf diff --git a/azure-observability/output.tf b/azure-collection-terraform/output.tf similarity index 100% rename from azure-observability/output.tf rename to azure-collection-terraform/output.tf diff --git a/azure-observability/providers.tf b/azure-collection-terraform/providers.tf similarity index 100% rename from azure-observability/providers.tf rename to azure-collection-terraform/providers.tf diff --git a/azure-observability/sumo_resources.tf b/azure-collection-terraform/sumo_resources.tf similarity index 100% rename from azure-observability/sumo_resources.tf rename to azure-collection-terraform/sumo_resources.tf diff --git a/azure-observability/variables.tf b/azure-collection-terraform/variables.tf similarity index 100% rename from azure-observability/variables.tf rename to azure-collection-terraform/variables.tf diff --git a/azure-observability/versions.tf b/azure-collection-terraform/versions.tf similarity index 100% rename from azure-observability/versions.tf rename to azure-collection-terraform/versions.tf