diff --git a/apps/otlp.go b/apps/otlp.go index a46181dae7..fd1f6309a6 100644 --- a/apps/otlp.go +++ b/apps/otlp.go @@ -145,7 +145,7 @@ func (r ReceiverOTLP) Pipelines(ctx context.Context) ([]otel.ReceiverPipeline, e ExporterTypes: map[string]otel.ExporterType{ "metrics": receiverPipelineType, "traces": otel.OTel, - "logs": otel.Logging, + "logs": otel.LoggingPersistentQueue, }, Receiver: otel.Component{ Type: "otlp", diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go index 679d66b50a..54ab56aad0 100644 --- a/confgenerator/confgenerator.go +++ b/confgenerator/confgenerator.go @@ -57,15 +57,34 @@ func googleCloudExporter(userAgent string, instrumentationLabels bool, serviceRe } } -func googleCloudLoggingExporter() otel.Component { - return otel.Component{ - Type: "googlecloud", - Config: map[string]interface{}{ - // Set to mirror the 60s max limit of default retry window in Google Cloud Logging apiv2 go client : - // https://github.com/googleapis/google-cloud-go/blob/logging/v1.4.2/logging/apiv2/logging_client.go#L78-L90 - "timeout": "60s", +func googleCloudLoggingExporter(persistent_queue bool) otel.Component { + config := map[string]interface{}{ + // Set to mirror the 60s max limit of default retry window in Google Cloud Logging apiv2 go client : + // https://github.com/googleapis/google-cloud-go/blob/logging/v1.4.2/logging/apiv2/logging_client.go#L78-L90 + "timeout": "60s", + "sending_queue": map[string]interface{}{ + "enabled": true, + "block_on_overflow": true, + "wait_for_result": true, + "batch": map[string]interface{}{ + "flush_timeout": "200ms", + "min_size": 1, + "max_size": 1000, + "sizer": "items", + }, }, } + + if persistent_queue { + if sq, ok := config["sending_queue"].(map[string]interface{}); ok { + sq["storage"] = dbStorageExtensionID() + } + } + + return otel.Component{ + Type: "googlecloud", + Config: config, + } } func ConvertPrometheusExporterToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline { @@ -152,6 +171,11 @@ func fileStorageExtensionID() string { return "file_storage" } +// dbStorageExtensionID returns the file_storage extension used by all receivers and exporters. +func dbStorageExtensionID() string { + return "db_storage" +} + // fileStorageExtensionConfig returns a configured file_storage extension to be used by all receivers and exporters. func fileStorageExtensionConfig(stateDir string) map[string]interface{} { return map[string]interface{}{ @@ -160,6 +184,14 @@ func fileStorageExtensionConfig(stateDir string) map[string]interface{} { } } +// dbStorageExtensionConfig returns a configured file_storage extension to be used by all receivers and exporters. +func dbStorageExtensionConfig(stateDir string) map[string]interface{} { + return map[string]interface{}{ + "driver": "sqlite", + "datasource": fmt.Sprintf("file://%s?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)", path.Join(stateDir, "db_storage.db")), + } +} + func (uc *UnifiedConfig) getEnabledExtensions(ctx context.Context, stateDir string) map[string]interface{} { extensions := map[string]interface{}{} expOtlpExporter := experimentsFromContext(ctx)["otlp_exporter"] @@ -168,6 +200,7 @@ func (uc *UnifiedConfig) getEnabledExtensions(ctx context.Context, stateDir stri } if uc.Logging.Service.OTelLogging { extensions["file_storage"] = fileStorageExtensionConfig(stateDir) + extensions["db_storage"] = dbStorageExtensionConfig(stateDir) } return extensions } @@ -222,14 +255,11 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDi }, }, }, - otel.Logging: { - Exporter: googleCloudLoggingExporter(), - ProcessorsByType: map[string][]otel.Component{ - // Batching logs improves log export performance. - "logs": { - otel.BatchProcessor(1000, 1000, "200ms"), - }, - }, + otel.LoggingPersistentQueue: { + Exporter: googleCloudLoggingExporter(true), + }, + otel.LoggingNonPersistentQueue: { + Exporter: googleCloudLoggingExporter(false), }, }, }.Generate(ctx) diff --git a/confgenerator/logging_receivers.go b/confgenerator/logging_receivers.go index bdd33b05c6..20982ae725 100644 --- a/confgenerator/logging_receivers.go +++ b/confgenerator/logging_receivers.go @@ -244,7 +244,7 @@ func (r LoggingReceiverFilesMixin) Pipelines(ctx context.Context) ([]otel.Receiv "logs": nil, }, ExporterTypes: map[string]otel.ExporterType{ - "logs": otel.Logging, + "logs": otel.LoggingNonPersistentQueue, }, }}, nil } @@ -369,7 +369,7 @@ func (r LoggingReceiverSyslog) Pipelines(ctx context.Context) ([]otel.ReceiverPi }, ExporterTypes: map[string]otel.ExporterType{ - "logs": otel.Logging, + "logs": otel.LoggingPersistentQueue, }, }}, nil } @@ -638,7 +638,7 @@ func (r LoggingReceiverWindowsEventLog) Pipelines(ctx context.Context) ([]otel.R "logs": p, }, ExporterTypes: map[string]otel.ExporterType{ - "logs": otel.Logging, + "logs": otel.LoggingNonPersistentQueue, }, }) } @@ -1028,7 +1028,7 @@ func (r LoggingReceiverSystemd) Pipelines(ctx context.Context) ([]otel.ReceiverP }, ExporterTypes: map[string]otel.ExporterType{ - "logs": otel.Logging, + "logs": otel.LoggingNonPersistentQueue, }, }}, nil } diff --git a/confgenerator/otel/modular.go b/confgenerator/otel/modular.go index d2a738383a..d2c257934f 100644 --- a/confgenerator/otel/modular.go +++ b/confgenerator/otel/modular.go @@ -18,6 +18,7 @@ package otel import ( "context" "fmt" + "slices" "github.com/GoogleCloudPlatform/ops-agent/internal/platform" yaml "github.com/goccy/go-yaml" @@ -38,7 +39,8 @@ const ( System GMP OTLP - Logging + LoggingPersistentQueue + LoggingNonPersistentQueue ) const ( Override ResourceDetectionMode = iota @@ -52,8 +54,10 @@ func (t ExporterType) Name() string { return "" } else if t == OTel { return "otel" - } else if t == Logging { - return "logging" + } else if t == LoggingPersistentQueue { + return "logging_persistent_queue" + } else if t == LoggingNonPersistentQueue { + return "logging_non_persistent_queue" } else if t == OTLP { return "otlp" } else { @@ -209,6 +213,7 @@ func (c ModularConfig) Generate(ctx context.Context) (string, error) { extensions[extensionName] = c.Extensions[extensionName] extensionsList = append(extensionsList, extensionName) } + slices.Sort(extensionsList) service["extensions"] = extensionsList configMap["extensions"] = extensions } diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml index 822cdbd3dc..bfd958cc69 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml @@ -7,7 +7,17 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s googlecloud/otel: metric: @@ -25,10 +35,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -738,11 +744,10 @@ service: pipelines: logs/logs_otlp_otlp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/otlp_0 - resourcedetection/_global_1 - - batch/googlecloud/logging_logs_0 receivers: - otlp/otlp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml index e8c79bff12..7a03b526cf 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml @@ -7,7 +7,17 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s googlecloud/otel: metric: @@ -25,10 +35,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -707,11 +713,10 @@ service: pipelines: logs/logs_otlp_otlp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/otlp_0 - resourcedetection/_global_1 - - batch/googlecloud/logging_logs_0 receivers: - otlp/otlp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml index a33a37df22..b884dde2e7 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml @@ -7,7 +7,17 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s googlecloud/otel: metric: @@ -25,10 +35,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -820,11 +826,10 @@ service: pipelines: logs/logs_otlp_otlp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/otlp_0 - resourcedetection/_global_1 - - batch/googlecloud/logging_logs_0 receivers: - otlp/otlp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml index a33a37df22..b884dde2e7 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml @@ -7,7 +7,17 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s googlecloud/otel: metric: @@ -25,10 +35,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -820,11 +826,10 @@ service: pipelines: logs/logs_otlp_otlp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/otlp_0 - resourcedetection/_global_1 - - batch/googlecloud/logging_logs_0 receivers: - otlp/otlp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml index d3866f44c2..e1db0ddef5 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -875,20 +883,20 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/sample__logs_0 - filter/logs_p1_sample__logs_0 @@ -915,7 +923,6 @@ service: - filter/logs_p1_sample__logs_21 - filter/logs_p1_sample__logs_22 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml index 0f34224e79..4fe87d314e 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -844,20 +852,20 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/sample__logs_0 - filter/logs_p1_sample__logs_0 @@ -884,7 +892,6 @@ service: - filter/logs_p1_sample__logs_21 - filter/logs_p1_sample__logs_22 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml index 259a3ba414..0a1c5f9d6a 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1297,44 +1305,42 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/sample__logs_0 - filter/logs_p1_sample__logs_0 @@ -1361,7 +1367,6 @@ service: - filter/logs_p1_sample__logs_21 - filter/logs_p1_sample__logs_22 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml index 259a3ba414..0a1c5f9d6a 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1297,44 +1305,42 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/sample__logs_0 - filter/logs_p1_sample__logs_0 @@ -1361,7 +1367,6 @@ service: - filter/logs_p1_sample__logs_21 - filter/logs_p1_sample__logs_22 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml index 25f41a2ae5..b25b179ccc 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -848,25 +856,24 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml index 69d9db3c61..37c98b5db5 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -817,25 +825,24 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml index 6dd1ba29b6..6d5e66a1f5 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1270,49 +1278,46 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml index 6dd1ba29b6..6d5e66a1f5 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1270,49 +1278,46 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml index d8ef662f48..6d9b6a79f2 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml @@ -7,9 +7,33 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true + timeout: 60s + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +41,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -764,25 +784,24 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_1 - - batch/googlecloud/logging_logs_0 receivers: - otlp/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml index 07991c2beb..9d4ee4d64d 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml @@ -7,9 +7,33 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true + timeout: 60s + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +41,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -733,25 +753,24 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_1 - - batch/googlecloud/logging_logs_0 receivers: - otlp/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml index cd0d51dd91..6c619905e7 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml @@ -7,9 +7,33 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true + timeout: 60s + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +41,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1186,49 +1206,46 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_1 - - batch/googlecloud/logging_logs_0 receivers: - otlp/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml index cd0d51dd91..6c619905e7 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml @@ -7,9 +7,33 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true + timeout: 60s + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +41,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1186,49 +1206,46 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_1 - - batch/googlecloud/logging_logs_0 receivers: - otlp/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml index f7c46a3cea..2a700f2b88 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -815,25 +823,24 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/log__source__id1_0 - transform/logs_pipeline1_log__source__id1_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml index 24b15719d8..eea28e9357 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -784,25 +792,24 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/log__source__id1_0 - transform/logs_pipeline1_log__source__id1_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml index 2aaa7a632d..bda45afa87 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1237,49 +1245,46 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/log__source__id1_0 - transform/logs_pipeline1_log__source__id1_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml index 2aaa7a632d..bda45afa87 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1237,49 +1245,46 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/log__source__id1_0 - transform/logs_pipeline1_log__source__id1_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml index 348b453b51..3112d88316 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -771,16 +779,16 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - transform/logs_default__pipeline_syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml index 7421e42482..ae319a921b 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -740,16 +748,16 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - transform/logs_default__pipeline_syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml index d927eb8d72..aabac6041a 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1303,42 +1311,40 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - transform/logs_default__pipeline_windows__event__log_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - transform/logs_default__pipeline_windows__event__log_1_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - transform/logs_default__pipeline_windows__event__log_2_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml index d927eb8d72..aabac6041a 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1303,42 +1311,40 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - transform/logs_default__pipeline_windows__event__log_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - transform/logs_default__pipeline_windows__event__log_1_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - transform/logs_default__pipeline_windows__event__log_2_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml index 5d87bb6cf1..1d36c9c09b 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -760,24 +768,23 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/log__source__id1_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml index 723c4c60d4..ced37dddd1 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -729,24 +737,23 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/log__source__id1_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml index 72d312ccb1..e00371322c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1182,48 +1190,45 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/log__source__id1_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml index 72d312ccb1..e00371322c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1182,48 +1190,45 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/log__source__id1_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml index 70129b80f6..6c3708a695 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -850,27 +858,26 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_kafka_kafka: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/kafka_0 - transform/kafka_1 - transform/kafka_2 - transform/kafka_3 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/kafka metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml index 6fcc9d8591..4569dd1720 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -819,27 +827,26 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_kafka_kafka: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/kafka_0 - transform/kafka_1 - transform/kafka_2 - transform/kafka_3 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/kafka metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml index 8224db04a7..792ff31ad1 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1272,51 +1280,48 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_kafka_kafka: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/kafka_0 - transform/kafka_1 - transform/kafka_2 - transform/kafka_3 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/kafka metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml index 8224db04a7..792ff31ad1 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1272,51 +1280,48 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_kafka_kafka: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/kafka_0 - transform/kafka_1 - transform/kafka_2 - transform/kafka_3 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/kafka metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/otel.yaml index f3329a2b6e..4a33c451c6 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -1390,20 +1398,20 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_mysql_mysql__error: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/mysql__error_0 - transform/mysql__error_1 @@ -1411,12 +1419,11 @@ service: - transform/mysql__error_3 - transform/mysql__error_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__error logs/logs_mysql_mysql__general: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/mysql__general_0 - transform/mysql__general_1 @@ -1424,12 +1431,11 @@ service: - transform/mysql__general_3 - transform/mysql__general_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__general logs/logs_mysql_mysql__slow: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/mysql__slow_0 - transform/mysql__slow_1 @@ -1437,7 +1443,6 @@ service: - transform/mysql__slow_3 - transform/mysql__slow_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__slow metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/otel.yaml index da5cb03d53..3275167af8 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -1359,20 +1367,20 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_mysql_mysql__error: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/mysql__error_0 - transform/mysql__error_1 @@ -1380,12 +1388,11 @@ service: - transform/mysql__error_3 - transform/mysql__error_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__error logs/logs_mysql_mysql__general: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/mysql__general_0 - transform/mysql__general_1 @@ -1393,12 +1400,11 @@ service: - transform/mysql__general_3 - transform/mysql__general_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__general logs/logs_mysql_mysql__slow: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/mysql__slow_0 - transform/mysql__slow_1 @@ -1406,7 +1412,6 @@ service: - transform/mysql__slow_3 - transform/mysql__slow_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__slow metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/otel.yaml index 8ec2a62ef3..e3dfc7a88b 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1812,44 +1820,42 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_mysql_mysql__error: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/mysql__error_0 - transform/mysql__error_1 @@ -1857,12 +1863,11 @@ service: - transform/mysql__error_3 - transform/mysql__error_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__error logs/logs_mysql_mysql__general: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/mysql__general_0 - transform/mysql__general_1 @@ -1870,12 +1875,11 @@ service: - transform/mysql__general_3 - transform/mysql__general_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__general logs/logs_mysql_mysql__slow: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/mysql__slow_0 - transform/mysql__slow_1 @@ -1883,7 +1887,6 @@ service: - transform/mysql__slow_3 - transform/mysql__slow_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__slow metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/otel.yaml index 8ec2a62ef3..e3dfc7a88b 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1812,44 +1820,42 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_mysql_mysql__error: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/mysql__error_0 - transform/mysql__error_1 @@ -1857,12 +1863,11 @@ service: - transform/mysql__error_3 - transform/mysql__error_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__error logs/logs_mysql_mysql__general: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/mysql__general_0 - transform/mysql__general_1 @@ -1870,12 +1875,11 @@ service: - transform/mysql__general_3 - transform/mysql__general_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__general logs/logs_mysql_mysql__slow: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - logstransform/mysql__slow_0 - transform/mysql__slow_1 @@ -1883,7 +1887,6 @@ service: - transform/mysql__slow_3 - transform/mysql__slow_4 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/mysql__slow metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml index acdba612cf..1c8273a930 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -1022,37 +1030,35 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_nginx_nginx__access: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/nginx__access_0 - transform/nginx__access_1 - transform/nginx__access_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/nginx__access logs/logs_nginx_nginx__error: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/nginx__error_0 - transform/nginx__error_1 - transform/nginx__error_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/nginx__error metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml index 16cd2a4677..28ba8b6ee4 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -991,37 +999,35 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_nginx_nginx__access: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/nginx__access_0 - transform/nginx__access_1 - transform/nginx__access_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/nginx__access logs/logs_nginx_nginx__error: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/nginx__error_0 - transform/nginx__error_1 - transform/nginx__error_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/nginx__error metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml index 0d41b2b2c9..770ab2f631 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1444,61 +1452,57 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_nginx_nginx__access: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/nginx__access_0 - transform/nginx__access_1 - transform/nginx__access_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/nginx__access logs/logs_nginx_nginx__error: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/nginx__error_0 - transform/nginx__error_1 - transform/nginx__error_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/nginx__error metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml index 0d41b2b2c9..770ab2f631 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1444,61 +1452,57 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_nginx_nginx__access: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/nginx__access_0 - transform/nginx__access_1 - transform/nginx__access_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/nginx__access logs/logs_nginx_nginx__error: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/nginx__error_0 - transform/nginx__error_1 - transform/nginx__error_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/nginx__error metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml index 389a2d80cd..b0218b3afa 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml @@ -7,9 +7,22 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +30,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -874,28 +883,27 @@ receivers: listen_address: 2.2.2.2:2222 service: extensions: + - db_storage - file_storage pipelines: logs/logs_pipeline1_test__syslog__source__id__tcp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/test__syslog__source__id__tcp_0 - transform/test__syslog__source__id__tcp_1 - transform/logs_pipeline1_test__syslog__source__id__tcp_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - syslog/test__syslog__source__id__tcp logs/logs_pipeline2_test__syslog__source__id__udp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/test__syslog__source__id__udp_0 - transform/test__syslog__source__id__udp_1 - transform/logs_pipeline2_test__syslog__source__id__udp_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - syslog/test__syslog__source__id__udp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml index 687f3cf6c7..f67a0e304b 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml @@ -7,9 +7,22 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +30,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -843,28 +852,27 @@ receivers: listen_address: 2.2.2.2:2222 service: extensions: + - db_storage - file_storage pipelines: logs/logs_pipeline1_test__syslog__source__id__tcp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/test__syslog__source__id__tcp_0 - transform/test__syslog__source__id__tcp_1 - transform/logs_pipeline1_test__syslog__source__id__tcp_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - syslog/test__syslog__source__id__tcp logs/logs_pipeline2_test__syslog__source__id__udp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/test__syslog__source__id__udp_0 - transform/test__syslog__source__id__udp_1 - transform/logs_pipeline2_test__syslog__source__id__udp_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - syslog/test__syslog__source__id__udp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml index 7e9a032dda..846b88cc64 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml @@ -7,9 +7,22 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +30,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -956,28 +965,27 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_pipeline1_test__syslog__source__id__tcp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/test__syslog__source__id__tcp_0 - transform/test__syslog__source__id__tcp_1 - transform/logs_pipeline1_test__syslog__source__id__tcp_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - syslog/test__syslog__source__id__tcp logs/logs_pipeline2_test__syslog__source__id__udp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/test__syslog__source__id__udp_0 - transform/test__syslog__source__id__udp_1 - transform/logs_pipeline2_test__syslog__source__id__udp_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - syslog/test__syslog__source__id__udp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml index 7e9a032dda..846b88cc64 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml @@ -7,9 +7,22 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + storage: db_storage + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +30,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -956,28 +965,27 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_pipeline1_test__syslog__source__id__tcp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/test__syslog__source__id__tcp_0 - transform/test__syslog__source__id__tcp_1 - transform/logs_pipeline1_test__syslog__source__id__tcp_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - syslog/test__syslog__source__id__tcp logs/logs_pipeline2_test__syslog__source__id__udp: exporters: - - googlecloud/logging + - googlecloud/logging_persistent_queue processors: - transform/test__syslog__source__id__udp_0 - transform/test__syslog__source__id__udp_1 - transform/logs_pipeline2_test__syslog__source__id__udp_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - syslog/test__syslog__source__id__udp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml index 50e270c292..e9202447b5 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -785,25 +793,24 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_systemd__pipeline_systemd__logs: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/systemd__logs_0 - transform/systemd__logs_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - journald/systemd__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml index 63bd352092..3d3cbb6739 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -754,25 +762,24 @@ receivers: - 0.0.0.0:20201 service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/syslog_0 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - filelog/syslog logs/logs_systemd__pipeline_systemd__logs: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/systemd__logs_0 - transform/systemd__logs_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - journald/systemd__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml index 95eeac7ae1..5257abe8c6 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1564,69 +1572,64 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_winlog2_winlog2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/winlog2_0 - transform/winlog2_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/winlog2 logs/logs_winlog2_winlog2_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/winlog2_1_0 - transform/winlog2_1_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/winlog2_1 logs/logs_winlog2_winlog2_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/winlog2_2_0 - transform/winlog2_2_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/winlog2_2 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml index 95eeac7ae1..5257abe8c6 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1564,69 +1572,64 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_winlog2_winlog2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/winlog2_0 - transform/winlog2_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/winlog2 logs/logs_winlog2_winlog2_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/winlog2_1_0 - transform/winlog2_1_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/winlog2_1 logs/logs_winlog2_winlog2_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/winlog2_2_0 - transform/winlog2_2_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/winlog2_2 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml index c6f7ec7a2e..4353c11a41 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1262,59 +1270,55 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_winlog2_winlog2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/winlog2_0 - transform/winlog2_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/winlog2 logs/logs_winlog2_winlog2_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/winlog2_1_0 - transform/winlog2_1_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/winlog2_1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml index c6f7ec7a2e..4353c11a41 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml @@ -7,9 +7,21 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/logging: + googlecloud/logging_non_persistent_queue: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1 + sizer: items + block_on_overflow: true + enabled: true + wait_for_result: true timeout: 60s extensions: + db_storage: + datasource: file://db_storage.db?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL) + driver: sqlite file_storage: create_directory: true directory: file_storage @@ -17,10 +29,6 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization - batch/googlecloud/logging_logs_0: - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 200ms casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1262,59 +1270,55 @@ receivers: object: SQLServer:Databases service: extensions: + - db_storage - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/windows__event__log_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/windows__event__log_1_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/windows__event__log_2_2 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/windows__event__log_2 logs/logs_winlog2_winlog2: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/winlog2_0 - transform/winlog2_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/winlog2 logs/logs_winlog2_winlog2_1: exporters: - - googlecloud/logging + - googlecloud/logging_non_persistent_queue processors: - transform/winlog2_1_0 - transform/winlog2_1_1 - resourcedetection/_global_0 - - batch/googlecloud/logging_logs_0 receivers: - windowseventlog/winlog2_1 metrics/default__pipeline_hostmetrics: diff --git a/transformation_test/testdata/ops_agent_test-TestLogEntrySpecialFields/output_otel.yaml b/transformation_test/testdata/ops_agent_test-TestLogEntrySpecialFields/output_otel.yaml index a59fec836d..59c2bc358e 100644 --- a/transformation_test/testdata/ops_agent_test-TestLogEntrySpecialFields/output_otel.yaml +++ b/transformation_test/testdata/ops_agent_test-TestLogEntrySpecialFields/output_otel.yaml @@ -3,7 +3,7 @@ error: "could not process attribute gcp.source_location: json: cannot unmarshal string into Go struct field LogEntrySourceLocation.line of type int64" level: error msg: Exporting failed. Rejecting data. Try enabling sending_queue to survive temporary failures. - otelcol.component.id: googlecloud/logging + otelcol.component.id: googlecloud/logging_non_persistent_queue otelcol.component.kind: exporter otelcol.signal: logs rejected_items: 1.0 @@ -32,7 +32,7 @@ error: "could not process attribute gcp.source_location: json: cannot unmarshal string into Go struct field LogEntrySourceLocation.line of type int64" level: warn msg: Sender failed - otelcol.component.id: batch/googlecloud/logging_logs_0 + otelcol.component.id: batch/googlecloud/logging_non_persistent_queue_logs_0 otelcol.component.kind: processor otelcol.pipeline.id: logs/logs_transformation__test_my-log-name otelcol.signal: logs diff --git a/transformation_test/transformation_test.go b/transformation_test/transformation_test.go index 689049a32e..584e4edcc8 100644 --- a/transformation_test/transformation_test.go +++ b/transformation_test/transformation_test.go @@ -404,7 +404,7 @@ func (transformationConfig transformationTest) generateOTelConfig(ctx context.Co ReceiverPipelines: rps, Pipelines: pls, Exporters: map[otel.ExporterType]otel.ExporterComponents{ - otel.Logging: { + otel.LoggingNonPersistentQueue: { ProcessorsByType: map[string][]otel.Component{ // Batch with 1.5s timeout to group in the same log request // all late entries flushed from a multiline parser after 1s.