[confgenerator] Implement LoggingReceiverTCP in Otel Logging.

confgenerator/logging_receivers.go

@@ -430,6 +430,52 @@ func (r LoggingReceiverTCP) Components(ctx context.Context, tag string) []fluent
 	}}
 }
 
+func (r LoggingReceiverTCP) Pipelines(ctx context.Context) ([]otel.ReceiverPipeline, error) {
+	// body := ottl.LValue{"body"}
+	// bodyMessage := ottl.LValue{"body", "message"}
+	// bodyLog := ottl.LValue{"body", "log"}
+	// attributes := ottl.LValue{"attributes"}
+	// cacheBodyString := ottl.LValue{"cache", "body_string"}
+	// cacheBodyMap := ottl.LValue{"cache", "body_map"}
+
+	// processors := []otel.Component{
+	// 	otel.Transform(
+	// 		"log", "log",
+	// 		// Transformations required to convert "fluentforwardreceiver" output to the expected ops agent "fluent_forward" LogEntry format.
+	// 		// In summary, this moves all resulting "fluentforwardreceiver" fields into "body" (jsonPayload).
+	// 		// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/release/v0.136.x/receiver/fluentforwardreceiver/conversion.go#L171
+	// 		ottl.NewStatements(
+	// 			// "fluentforwardreceiver" sets "log" and "message" as "body". All other fields are set as "attributes".
+	// 			cacheBodyString.SetIf(body, body.IsString()),
+	// 			cacheBodyMap.SetIf(body, body.IsMap()),
+	// 			// Merge "cache['body_string']", "cache['body_map']" and "attributes" into "body" (jsonPayload).
+	// 			body.Set(ottl.RValue("{}")),
+	// 			bodyMessage.SetIf(cacheBodyString, cacheBodyString.IsPresent()),
+	// 			bodyLog.SetIf(cacheBodyString, cacheBodyString.IsPresent()),
+	// 			body.MergeMapsIf(cacheBodyMap, "upsert", cacheBodyMap.IsPresent()),
+	// 			body.MergeMapsIf(attributes, "upsert", attributes.IsPresent()),
+	// 			attributes.Set(ottl.RValue("{}")),
+	// 		),
+	// 	),
+	// }
+
+	return []otel.ReceiverPipeline{{
+		Receiver: otel.Component{
+			Type: "tcplog",
+			Config: map[string]any{
+				"listen_address": fmt.Sprintf("%s:%d", r.ListenHost, r.ListenPort),
+			},
+		},
+		Processors: map[string][]otel.Component{
+			"logs": []otel.Component{},
+		},
+
+		ExporterTypes: map[string]otel.ExporterType{
+			"logs": otel.OTel,
+		},
+	}}, nil
+}
+
 func init() {
 	LoggingReceiverTypes.RegisterType(func() LoggingReceiver { return &LoggingReceiverTCP{} })
 }

confgenerator/testdata/goldens/logging-otel-receiver_tcp/EXPERIMENTAL_FEATURES

@@ -0,0 +1 @@
+otel_logging

confgenerator/testdata/goldens/logging-otel-receiver_tcp/golden/linux-gpu/0f15dbe303dc7122d43443c9a4c31632.lua

@@ -0,0 +1,28 @@
+
+function process(tag, timestamp, record)
+local v = "ops-agent";
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["agent.googleapis.com/health/agentKind"] = value
+end)(v)
+local v = "latest";
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["agent.googleapis.com/health/agentVersion"] = value
+end)(v)
+local v = "v1";
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["agent.googleapis.com/health/schemaVersion"] = value
+end)(v)
+return 2, timestamp, record
+end

confgenerator/testdata/goldens/logging-otel-receiver_tcp/golden/linux-gpu/4d6012ff003886818fb9b9285b4af962.lua

@@ -0,0 +1,19 @@
+
+function process(tag, timestamp, record)
+local __field_0 = (function()
+return record["severity"]
+end)();
+(function(value)
+record["severity"] = value
+end)(nil);
+local v = __field_0;
+if v == "debug" then v = "DEBUG"
+elseif v == "error" then v = "ERROR"
+elseif v == "info" then v = "INFO"
+elseif v == "warn" then v = "WARNING"
+end
+(function(value)
+record["logging.googleapis.com/severity"] = value
+end)(v)
+return 2, timestamp, record
+end

confgenerator/testdata/goldens/logging-otel-receiver_tcp/golden/linux-gpu/5fc5f42c16c9e1ab8292e3d42f74f3be.lua

@@ -0,0 +1,49 @@
+
+  function shallow_merge(record, parsedRecord)
+    -- If no exiting record exists
+    if (record == nil) then 
+        return parsedRecord
+    end
+
+    for k, v in pairs(parsedRecord) do
+        record[k] = v
+    end
+
+    return record
+end
+
+function merge(record, parsedRecord)
+    -- If no exiting record exists
+    if record == nil then 
+        return parsedRecord
+    end
+
+    -- Potentially overwrite or merge the original records.
+    for k, v in pairs(parsedRecord) do
+        -- If there is no conflict
+        if k == "logging.googleapis.com/logName" then 
+            -- Ignore the parsed payload since the logName is controlled
+            -- by the OpsAgent.
+        elseif k == "logging.googleapis.com/labels" then 
+            -- LogEntry.labels are basically a map[string]string and so only require a
+            -- shallow merge (one level deep merge).
+            record[k] = shallow_merge(record[k], v)
+        else
+            record[k] = v
+        end
+    end
+
+    return record
+end
+
+function parser_merge_record(tag, timestamp, record)
+    originalPayload = record["logging.googleapis.com/__tmp"]
+    if originalPayload == nil then
+        return 0, timestamp, record
+    end
+
+    -- Remove original payload
+    record["logging.googleapis.com/__tmp"] = nil
+    record = merge(originalPayload, record)
+    return 2, timestamp, record
+end

confgenerator/testdata/goldens/logging-otel-receiver_tcp/golden/linux-gpu/b4a0dead382dce7b4fe011d3f59fdb6d.lua

@@ -0,0 +1,17 @@
+
+function parser_nest(tag, timestamp, record)
+  local nestedRecord = {}
+  local parseKey = "message"
+  for k, v in pairs(record) do
+      if k ~= parseKey then
+          nestedRecord[k] = v
+      end
+  end
+
+  local result = {}
+  result[parseKey] = record[parseKey]
+  result["logging.googleapis.com/__tmp"] = nestedRecord
+
+  return 2, timestamp, result
+end
+

confgenerator/testdata/goldens/logging-otel-receiver_tcp/golden/linux-gpu/enabled_receivers_otlp.json

@@ -0,0 +1 @@
+{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/ops_agent/enabled_receivers","gauge":{"dataPoints":[{"attributes":[{"key":"telemetry_type","value":{"stringValue":"metrics"}},{"key":"receiver_type","value":{"stringValue":"hostmetrics"}}],"asInt":"1"},{"attributes":[{"key":"telemetry_type","value":{"stringValue":"logs"}},{"key":"receiver_type","value":{"stringValue":"files"}}],"asInt":"1"},{"attributes":[{"key":"telemetry_type","value":{"stringValue":"logs"}},{"key":"receiver_type","value":{"stringValue":"tcp"}}],"asInt":"1"}]}}]}]}]}

confgenerator/testdata/goldens/logging-otel-receiver_tcp/golden/linux-gpu/feature_tracking_otlp.json

@@ -0,0 +1 @@
+{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:tcp"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]}

confgenerator/testdata/goldens/logging-otel-receiver_tcp/golden/linux-gpu/features.yaml

@@ -0,0 +1,20 @@
+- module: logging
+  feature: service:pipelines
+  key: default_pipeline_overridden
+  value: "false"
+- module: metrics
+  feature: service:pipelines
+  key: default_pipeline_overridden
+  value: "false"
+- module: global
+  feature: default:self_log
+  key: default_self_log_file_collection
+  value: "true"
+- module: logging
+  feature: service:otel_logging
+  key: otel_logging_supported_config
+  value: "true"
+- module: logging
+  feature: receivers:tcp
+  key: "[0].enabled"
+  value: "true"

confgenerator/testdata/goldens/logging-otel-receiver_tcp/golden/linux-gpu/fluent_bit_main.conf

@@ -0,0 +1,153 @@
+@SET buffers_dir=/var/lib/google-cloud-ops-agent/fluent-bit/buffers
+@SET logs_dir=/var/log/google-cloud-ops-agent
+
+[SERVICE]
+    Daemon                    off
+    Flush                     1
+    Log_Level                 info
+    dns.resolver              legacy
+    storage.backlog.mem_limit 50M
+    storage.checksum          off
+    storage.max_chunks_up     128
+    storage.metrics           on
+    storage.sync              normal
+
+[INPUT]
+    Name            fluentbit_metrics
+    Scrape_Interval 60
+    Scrape_On_Start True
+
+[INPUT]
+    Dummy         {"code": "LogPingOpsAgent", "severity": "DEBUG"}
+    Interval_NSec 0
+    Interval_Sec  600
+    Name          dummy
+    Tag           ops-agent-health
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   2M
+    DB                ${buffers_dir}/ops-agent-fluent-bit
+    DB.locking        true
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              ${logs_dir}/subagents/logging-module.log
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               ops-agent-fluent-bit
+    storage.type      memory
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   2M
+    DB                ${buffers_dir}/ops-agent-health
+    DB.locking        true
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              ${logs_dir}/health-checks.log
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               ops-agent-health
+    storage.type      memory
+
+[FILTER]
+    Match  ops-agent-fluent-bit
+    Name   lua
+    call   parser_nest
+    script b4a0dead382dce7b4fe011d3f59fdb6d.lua
+
+[FILTER]
+    Key_Name     message
+    Match        ops-agent-fluent-bit
+    Name         parser
+    Preserve_Key True
+    Reserve_Data True
+    Parser       ops-agent-fluent-bit.fluent-bit-self-log-regex-parsing
+
+[FILTER]
+    Match  ops-agent-fluent-bit
+    Name   lua
+    call   parser_merge_record
+    script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua
+
+[FILTER]
+    Match  ops-agent-health
+    Name   lua
+    call   parser_nest
+    script b4a0dead382dce7b4fe011d3f59fdb6d.lua
+
+[FILTER]
+    Key_Name     message
+    Match        ops-agent-health
+    Name         parser
+    Reserve_Data True
+    Parser       ops-agent-health.health-checks-json
+
+[FILTER]
+    Match  ops-agent-health
+    Name   lua
+    call   parser_merge_record
+    script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua
+
+[FILTER]
+    Match ops-agent-health
+    Name  grep
+    Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug
+
+[FILTER]
+    Match ops-agent-fluent-bit
+    Name  rewrite_tag
+    Rule  message \[error\]\s\[lib\]\sbackend\sfailed ops-agent-health true
+
+[FILTER]
+    Name      modify
+    Match     ops-agent-health
+    Condition Key_value_matches message \[error\]\s\[lib\]\sbackend\sfailed
+    Set       code LogPipelineErr
+    Set       message "[Runtime Check] Result: FAIL, Error code: LogPipelineErr, Failure: Ops Agent logging pipeline failed, Solution: Refer to provided documentation link., Resource: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-find-info"
+
+[FILTER]
+    Match ops-agent-fluent-bit
+    Name  rewrite_tag
+    Rule  message \[error\]\s\[parser\]\scannot\sparse ops-agent-health true
+
+[FILTER]
+    Name      modify
+    Match     ops-agent-health
+    Condition Key_value_matches message \[error\]\s\[parser\]\scannot\sparse
+    Set       code LogParseErr
+    Set       message "[Runtime Check] Result: WARNING, Error code: LogParseErr, Failure: Ops Agent failed to parse logs, Solution: Refer to provided documentation link., Resource: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-find-info"
+
+[FILTER]
+    Match  ops-agent-health
+    Name   lua
+    call   process
+    script 0f15dbe303dc7122d43443c9a4c31632.lua
+
+[FILTER]
+    Match  ops-agent-*
+    Name   lua
+    call   process
+    script 4d6012ff003886818fb9b9285b4af962.lua
+
+[OUTPUT]
+    Match_Regex                   ^(ops-agent-health|ops-agent-fluent-bit)$
+    Name                          stackdriver
+    Retry_Limit                   3
+    http_request_key              logging.googleapis.com/httpRequest
+    net.connect_timeout_log_error False
+    resource                      gce_instance
+    stackdriver_agent             Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+    tls                           On
+    tls.verify                    Off
+    workers                       8
+
+[OUTPUT]
+    Match *
+    Name  prometheus_exporter
+    host  0.0.0.0
+    port  20202

confgenerator/testdata/goldens/logging-otel-receiver_tcp/golden/linux-gpu/fluent_bit_parser.conf

@@ -0,0 +1,13 @@
+[PARSER]
+    Format      regex
+    Name        ops-agent-fluent-bit.fluent-bit-self-log-regex-parsing
+    Regex       (?<message>\[[ ]*(?<time>\d+\/\d+\/\d+ \d+:\d+:\d+)(?:\.\d+)?\] \[[ ]*(?<severity>[a-z]+)\].*)
+    Time_Format %Y/%m/%d %H:%M:%S
+    Time_Key    time
+    Types       severity:string
+
+[PARSER]
+    Format      json
+    Name        ops-agent-health.health-checks-json
+    Time_Format %Y-%m-%dT%H:%M:%S%z
+    Time_Key    time