Skip to content

Enable persistent logs #2233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
quentinmit wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Enable persistent logs #2233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9a3c351
feat: Persist logs on disk to survive network outages
quentinmit Mar 11, 2026
f79df48
Update goldens
quentinmit Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions confgenerator/confgenerator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,8 @@ func googleCloudLoggingExporter() otel.Component {
return otel.Component{
Type: "googlecloud",
Config: map[string]interface{}{
// Set to mirror the 60s max limit of default retry window in Google Cloud Logging apiv2 go client :
// https://github.com/googleapis/google-cloud-go/blob/logging/v1.4.2/logging/apiv2/logging_client.go#L78-L90
"timeout": "60s",
// Keep trying to send log entries for 1 hour before we drop them.
"timeout": "3600s",
"sending_queue": map[string]interface{}{
"enabled": true,
// Set queue_size to "(num_consumers + 2)*1000" to always have a new batch ready.
Expand All @@ -79,6 +78,8 @@ func googleCloudLoggingExporter() otel.Component {
"max_size": 1000,
"sizer": "items",
},
// Persist logs on disk to survive restarts during network outages.
"storage": fileStorageExtensionID,
},
},
}
Expand Down Expand Up @@ -164,10 +165,9 @@ func (uc *UnifiedConfig) getOTelLogLevel() string {
return logLevel
}

// fileStorageExtensionID returns the file_storage extension used by all receivers and exporters.
func fileStorageExtensionID() string {
return "file_storage"
}
const (
fileStorageExtensionID = "file_storage"
)

// fileStorageExtensionConfig returns a configured file_storage extension to be used by all receivers and exporters.
func fileStorageExtensionConfig(stateDir string) map[string]interface{} {
Expand All @@ -184,7 +184,7 @@ func (uc *UnifiedConfig) getEnabledExtensions(ctx context.Context, stateDir stri
extensions["googleclientauth"] = map[string]interface{}{}
}
if uc.Logging.Service.OTelLogging {
extensions["file_storage"] = fileStorageExtensionConfig(stateDir)
extensions[fileStorageExtensionID] = fileStorageExtensionConfig(stateDir)
}
return extensions
}
Expand Down
6 changes: 3 additions & 3 deletions confgenerator/logging_receivers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ func (r LoggingReceiverFilesMixin) Pipelines(ctx context.Context) ([]otel.Receiv
"fingerprint_size": "5kb",
}
if !r.TransformationTest {
receiver_config["storage"] = fileStorageExtensionID()
receiver_config["storage"] = fileStorageExtensionID
}
if i := r.WildcardRefreshInterval; i != nil {
receiver_config["poll_interval"] = i.String()
Expand Down Expand Up @@ -650,7 +650,7 @@ func (r LoggingReceiverWindowsEventLog) Pipelines(ctx context.Context) ([]otel.R
"start_at": "beginning",
"poll_interval": "1s",
"ignore_channel_errors": true,
"storage": fileStorageExtensionID(),
"storage": fileStorageExtensionID,
// When "include_log_record_original = true", the event original XML string is set in `attributes."log.record.original"`.
"include_log_record_original": true,
}
Expand Down Expand Up @@ -1036,7 +1036,7 @@ func (r LoggingReceiverSystemd) Pipelines(ctx context.Context) ([]otel.ReceiverP
receiver_config := map[string]any{
"start_at": "beginning",
"priority": "debug",
"storage": fileStorageExtensionID(),
"storage": fileStorageExtensionID,
}

modify_fields_processors, err := LoggingProcessorModifyFields{
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
googlecloud/otel:
metric:
instrumentation_library_labels: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
googlecloud/otel:
metric:
instrumentation_library_labels: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
googlecloud/otel:
metric:
instrumentation_library_labels: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
googlecloud/otel:
metric:
instrumentation_library_labels: true
Expand Down
3 changes: 2 additions & 1 deletion ...generator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...erator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...enerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...rator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...stdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...r/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...ata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...enerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...ns/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...oldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...dens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion .../testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...ator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...stdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion ...or/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
3 changes: 2 additions & 1 deletion confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ exporters:
num_consumers: 10
queue_size: 12000
sizer: items
timeout: 60s
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
Expand Down
Loading
Loading